diff options
34 files changed, 4569 insertions, 0 deletions
diff --git a/myrpki.rototill/Makefile b/myrpki.rototill/Makefile new file mode 100644 index 00000000..e828df71 --- /dev/null +++ b/myrpki.rototill/Makefile @@ -0,0 +1,39 @@ +# $Id$ + +all: schema.py + +lint: myrpki.xml schema.rng + xmllint --noout --relaxng schema.rng myrpki.xml + +schema.rng: schema.rnc + trang schema.rnc schema.rng + +schema.py: schema.rng + echo >$@ 'import lxml.etree' + echo >>$@ -n "myrpki = lxml.etree.RelaxNG(lxml.etree.fromstring('''" + cat >>$@ schema.rng + echo >>$@ "'''))" + +parse: myrpki.xml schema.py + python xml-parse-test.py + +clean: + rm -rf *.xml bpki.myrpki bpki.myirbe test screenlog.* + python sql-cleaner.py + +format: myrpki.xml + xmllint --format myrpki.xml + +graph: + find . -name .svn -prune -o -type d -name 'bpki.*' -print | while read b; do python ../scripts/x509-dot.py $$b | dot -T ps2 | ps2pdf - $$b/graph.pdf; done + +verify: + sh verify-bpki.sh + +backup: + python sql-dumper.py + tar cvvzf test.$$(TZ='' date +%Y.%m.%d.%H.%M.%S).tgz screenlog.* test backup.*.sql + rm backup.*.sql + +test: schema.py + python yamltest.py diff --git a/myrpki.rototill/PLAN b/myrpki.rototill/PLAN new file mode 100644 index 00000000..d7e1b4ed --- /dev/null +++ b/myrpki.rototill/PLAN @@ -0,0 +1,132 @@ +-*- Text -*- +$Id$ + +plan for new myrpki setup tools. need better name for them, among +other things. + +- dig out old proposed better names for myrpki and myirbe, use those. + bpki.myrpki => bpki.resources (or perhaps bpki/resources) + bpki.myirbe => bpki.servers + + bpki.resources is used for tls client and all cms in up-down and + object publication. + + bpki.servers is used for all tls server certs, and is also used for + tls client and all cms in left-right and publication control. + + had proposed new names for .py scripts, dig those out but will also + be creating a bunch of new scripts here to confuse the issue. + +- for the moment i'm completely ignoring security of the new setup + protcol. this clearly will not fly even in the short run, but let's + start by figuring out who needs to do what to whom before worrying + about how to prove that everybody is who they claim to be. + +- need a script to create new self (resource holding identity). this + creates bpki.resources, takes that and [myrpki]handle variable, + packages that as an xml blob. + +- send self xml blob to parent, current theory seems to be upload to a + web form. parent responds with a lot of the stuff currently in + parents.csv, and perhaps also with a hint about where to publish, + all packaged as an xml blob. well, maybe (also?) a link to + publication service that i can click on? + + current fields in parents.csv: + + # Syntax: <parent_handle> <service_uri> <cms_bpki_cert_filename> <https_bpki_cert_filename> <myhandle> <sia_base> + + <parent_handle> is a private matter, that probably turns into name + of the parent xml blob when i store it on disk, nobody else's + business what i call it + + <service_uri> and <myhandle> are supplied by parent, based on + whatever name parent uses for itself and what name it uses for me. + i've told it what i call myself, but parent probably has its own + name for me. + + <cms_bpki_cert_filename> and <https_bpki_cert_filename> parent just + supplies, nothing complex there (well, except for boostrap process + for rootd, but that's a separate mess). + + <sia_base> still makes my head hurt. not just me. + +- also send self xml blob to hosting, who may or may not be another + aspect of myself. hosting entity supplies (new? update of mine?) + xml blob containing bpki.servers and rpkid service url + ([myirbe]rpkid_base url value). will need to supply all of this to + any children. + + [myirbe]pubd_base makes my head hurt. why is <sia_base> in + parents.csv while pubd_base is in [myirbe] ? gah.... along with + everything else, this seems like really weird db normalization. + + kind of seems like <sia_base> goes with [myirbe]pubd_base and both + really need to be supplied by resource holder via negotiation with a + publication service. + +- a lot of the data for rpkid and pubd operator can be pulled from + config file, no need for separate configuration. perhaps in this + brave new world we lock people using this ui into one big config + file for everything so we can avoid making them type everything + three times? still some icky stuff with [rpkid], [irdbd], etc + sections linking to each other, not immediately obvious how to fix + that without losing generality in core daemons, but maybe inspect + these sections again and there will be some simple approach. + +- when answering setup request from child, we need to save xml blob + they sent us as replacement for a line that's now in children.csv. + well, but we also need expiration date, how fun. + + # Syntax: <child_handle> <validitydate> <bpki_cert_filename> + + <child_handle> becomes name of xml blob file: children/foo.xml or + whatever. it's a private matter except that we have to tell the + child what we picked so they can use it in up-down protocol. this + same name goes into service url we cons up for this child. + + <validitydate> we need to deduce, invent, or look up, somehow. in + real production we would have this on file as it's tied to contract + expiration. in testing we've just been saying "one year from + today". + + <bpki_cert_filename> is the child's bpki.resources cert, which they + just handed us, yay. + +- when answering setup request from publication client, we need to + save xml blob, same as parent saving child. + + # Syntax: <client_handle> <bpki_cert_filename> <sia_base> + + <client_handle> is what we call this client, name of xml blob file + we save, we get to chose it but we have to tell client what we chose + so they can use it. this same name goes into service url we cons up + for this client. + + <bpki_cert_filename> is the client's bpki.resources cert, which they + just handed us, yay. + + <sia_base> is making my head hurt yet again. this time it is what + we will allow this client to use, so it'd better not conflict with + any other publication client. oh, wait, i said this was zero + security at the moment, ok. + +- i should write converters from the current parents.csv, + children.csv, and pubclients.csv into the new format, both to + simplify migration and also as a clue as to what i've forgotten. + +- the above story is weak on publication setup. among other things, + the linkage between parent and repository that publishes things that + come from classes derived from that parent is weak, perhaps because + that linkage doesn't make any particular sense outside of rpkid's + object model. the real restriction in rpkid is that all the resource + classes derived from a single parent object share a repository + object. + +- oh yeah, and, unrelated to any of the above, i should check the + syntax restrictions on up-down resource class names, and perhaps + replace the numeric (last vestige of sql-derived identifiers) + resource class names we're using with resource class names extruded + from irdb. which might involve hacking left-right protocol, and + might create a uniqueness problem when publishing children under + self, but would be nice to get rid of the sql-derived ids. diff --git a/myrpki.rototill/POW b/myrpki.rototill/POW new file mode 120000 index 00000000..43fccd7b --- /dev/null +++ b/myrpki.rototill/POW @@ -0,0 +1 @@ +../pow/buildlib/POW
\ No newline at end of file diff --git a/myrpki.rototill/README b/myrpki.rototill/README new file mode 100644 index 00000000..3057fd0e --- /dev/null +++ b/myrpki.rototill/README @@ -0,0 +1,660 @@ +$Id$ + +INTRODUCTION + +The design of rpkid and friends assumes that certain tasks can be +thrown over the wall to the registry's back end operation. This was a +deliberate design decision to allow rpkid et al to remain independent +of existing database schema, business PKIs, and so forth that a +registry might already have. All very nice, but it leaves someone who +just wants to test the tools or who has no existing back end with a +fairly large programming project. The tools in this directory attempt +to fill that gap. + +This is a basic implementation of what a registry back end would need +to use rpkid and friends. These tools do not use every available +option, nor are they necessarily as efficient as possible. Large +registries will almost certainly want to roll their own tools, perhaps +using these as a starting point. Nevertheless, we hope that these +tools will at least provide a useful example. + +The primary tools here consist of two Python programs: myrpki.py and +myirbe.py. The first is for use by any entity that needs resources +allocated via the RPKI system, the second is for use by entities that +actually run copies of rpkid and its several supporting programs. + +The basic idea here is that a user who has resources maintains a set +of .csv files containing a text representation of the data needed by +the back-end, along with a configuration file containing other +parameters. The intent is that these be very simple files that are +easy to generate either by hand or as a dump from relational database, +spreadsheet, awk script, whatever works in your environment. Given +these files, the user then runs the myrpki.py script to extract the +relevant information and encode everything about its back end state +into a single .xml file, which the script writes out to disk. The +user then conveys this .xml file via some convenient means (PGP-signed +mail, USB key, dog-sled) to the operator of the rpkid engine that will +perform RPKI services on behalf of the user. + +The rpkid operator collects these .xml files from all the resource +holders it hosts, and feeds them all into the myirbe.py script, which +uses the data in the .xml files to populate the IRDB, create objects +in rpkid and pubd via the left-right and publication protocols, +etcetera. The script rewrites its input .xml files to contain any +updated information (eg, PKCS #10 requests for business signing +context certificates), so that the .xml file once again contains +everything that must be communicated between the rpkid operator and +hosted resource holder. + +The rpkid operator ships the updated .xml back to the user, who then +runs the myrpki.py script again to perform any necessary actions (eg, +issuing business signing context certificates given the PKCS #10 +request sent by myirbe.py), resulting in another update to the .xml +file, which the user then ships back to the rpkid operator. This +cycle repeats until nothing further needs to be changed. + +Note that, as certificates and CRLs have expiration and nextUpdate +values, a low-level cycle of updates passing between resource holder +and rpkid operator will be necessary as a part of steady state +operation. [The current version of these tools does not yet +regenerate these expiring objects, but fixing this will be a +relatively minor matter.] + +Since we assume that anybody who bothers to run rpkid is also a +resource holder, myirbe.py and myrpki.py can use the same +configuration file, and myirbe.py will run myrpki.py automatically if +the [myrpki] section of the configuration file is present. + +The third important file in this system is the configuration file for +myrpki.py and myirbe.py. This contains a number of sections, some of +which are for these scripts, others of which are for the OpenSSL +command line tool, which these scripts use do most of the certificate +work. The examples/ subdirectory contains a commented version of the +configuration file that explains the various parameters. + +myrpki.py deliberately does not use any libraries other than the ones +that ship with Python 2.5; in particular, it does not require any of +the other Python RPKI code. This is intentional, to minimize +portability issues for hosted resource holders. It does require a +reasonably current version of the OpenSSL command line tool, but the +version that is built as a side effect of building the rcynic relying +party tool is adequate if the system copy of this tool isn't. + +The .csv files read by myrpki.py can be anything that the Python "csv" +library understands. By default, they're in tab-delimited format +(because the author finds this easier to read than the comma-delimited +format), but this can be changed to fit local needs. + +Please note: tab-delimited CSV is a format defined by a certain +popular spreadsheet program, and is *not* the same as +whitespace-separated text. Tab characters are *punctuation*, and each +tab character indicates the division between two columns. Two tab +characters in a row indicates a separator, a blank cell, and another +separator, not one separator. The upshot of all this is that +attempting to make your columns line up prettily will not work as you +expect, you will end up with too many cells, some of them empty. + +A number of the fields in the configuration or CSV files involve +certificates. Some of these are built automatically, others must be +imported so that the scripts can cross-certify them. The certificates +you need to import are all self-signed BPKI trust anchor certificates +generated by other entities; you import them by specifying the name of +a file where you stored the BPKI certificate in question (in OpenSSL +"PEM" format). + +Keep reading, and don't panic. + +The default configuration file name is myrpki.conf. + +See examples/myrpki.conf for details on the variables that you can +(and in some cases must) set. + +See examples/*.csv for commented examples of the several CSV files. +Note that the comments themselves are not legal CSV, they're just +present to make it easier to understand the examples. + +GETTING STARTED -- OVERVIEW + +As explained above, the two basic programs are myrpki.py (for resource +holders) and myirbe.py (for rpkid operators); myirbe.py runs myrpki.py +automatically for a rpkid operator's own resources if myirbe.py finds +a [myrpki] section in its configuration file. + +Which process you need to follow to get started depends on whether you +are running rpkid yourself or will be hosted by somebody else. We +call the first case "self-hosted", because the software treats running +rpkid to handle resources that you yourself hold as if you are an rpkid +operator who is hosting an entity that happens to be yourself. + +"$top" in the following refers to wherever you put the +subvert-rpki.hactrn.net code. Once we have autoconf and "make +install" targets, this will be some system directory or another; for +now, it's wherever you checked out a copy of the code from the +subversion repository or unpacked a tarball of the code. + +GETTING STARTED -- HOSTED CASE + +The basic steps involved in getting started for a resource holder who +is being hosted by somebody else are: + +1) Obtain contact information and BPKI trust anchors from RPKI parents + and an RPKI publication service (see below for details). + +2) Write a configuration file (copy $top/myrpki/examples/myrpki.conf + and edit as needed). You can skip the sections associated with the + various daemons and their runtime control tools ([myirbe], [rpkid], + [irdbd], [pubd], [rootd], [irbe_cli]). You *do* need to configure + the [myrpki] section. + +3) Using $top/myrpki/examples/*.csv as a guide, create a set of CSV + files representing RPKI parents, RPKI children, resources to be + assigned to RPKI children, and ROAs to be generated once the + necessary RPKI certificates are available. Most of these CSV files + can be empty while first getting started, the only file that + absolutely must be populated is the file describing parents. + + You may choose to place your configuration file (which we will + refer to here as myrpki.conf) and your CSV files in their own + directory. The software doesn't really care. If you use absolute + names for all the filename entries in the configuration file and + CSV files, you can put the files wherever you like; if you use + relative names, they will be interpreted relative to the directory + in which you run the program that reads the file. + + [At some future date we may provide a default directory for + relative filenames such as /usr/local/etc/rpki, but the above + description holds for now.] + +4) Run myrpki.py to generate a BPKI trust anchor and collect all the + data from the configuration file, CSV files, and newly created BPKI + into a single XML file which can be shipped to the rpkid operator + who is hosting your resources. + +5) Send the XML file generated in step (4) to your rpkid operator. + +6) Wait for your rpkid operator to ship you back an updated XML file + containing a PKCS #10 certificate request for the BPKI signing + context (BSC) created by rpkid. + +7) Run myrpki.py again with the XML file received in step (6), to + issue the BSC certificate and update the XML file again to contain + the newly issued BSC certificate. + +8) Send the updated XML file back to your rpkid operator. + +At this point you're done with initial setup. You will need to run +myrpki.py again whenever you make any changes to your configuration +file or CSV files. [Once myrpki.py knows how to update BPKI CRLs, you +will also need to run myrpki.py periodically to keep your BPKI CRLs up +to date.] Any time you run myrpki.py, you should send the updated XML +file to your rpkid operator, who will [generally?] send you a further +updated XML file in response. + +GETTING STARTED -- SELF-HOSTED CASE + +The first few steps involved in getting started for a self-hosted +resource holder (that is, a resource holder that runs its own copy of +rpkid) are the same as in the hosted case above; after that the +process diverges. + +[As of the time at which these instructions were written, it had +become clear that there really should be an additional setup script +which automates much of the following. That script hasn't been +written yet, so for the moment this documents the setup process as it +stands now. Once that setup script has been written, these +instructions will be updated to match. In the meantime, please accept +the author's apologies for the tedious nature of the current setup +process.] + +The [current] steps are: + +1) Obtain contact information and BPKI trust anchors from RPKI parents + and an RPKI publication service (see below for details). + +2) Write a configuration file (copy examples/myrpki.conf and edit as + needed). You need to configure the [myrpki] and [myirbe] sections + as well as the sections associated with the daemons you will be + running ([rpkid], [irdbd], [irbe_cli]). You only need to configure + the [pubd] section if you intend to run your own publication + service: in general this is not recommended, because each + additional publication service in the RPKI universe places a small + additional burden on every relying party, since every relying party + has to download data from every publication service. In general + it's better to use an existing publication service operated by + somebody else (eg, your RPKI parent) if you can. In general most + cases you can leave the [rootd] section alone, as in most cases you + should not be running rootd. + +3) Using $top/myrpki/examples/*.csv as a guide, create a set of CSV + files representing RPKI parents, RPKI children, resources to be + assigned to RPKI children, and ROAs to be generated once the + necessary RPKI certificates are available. Most of these CSV files + can be empty while first getting started, the only file that + absolutely must be populated is the file describing parents. + + You may choose to place your configuration file (which we will + refer to here as myrpki.conf) and your CSV files in their own + directory. The software doesn't really care. If you use absolute + names for all the filename entries in the configuration file and + CSV files, you can put the files wherever you like; if you use + relative names, they will be interpreted relative to the directory + in which you run the program that reads the file. + + [At some future date we may provide a default directory for + relative filenames such as /usr/local/etc/rpki, but the above + description holds for now.] + +4) See rpkid/doc/Installation, and follow the basic installation + instructions there to build the RFC-3779-aware OpenSSL code and + associated Python extension module. + +5) Next, you need to set up the MySQL databases that rpkid et al will + use. The MySQL database, username, and password values all need to + match the ones you specified in myrpki.conf. There are two + different ways you can do this: + + a) You can use the setup-sql.py script, which prompts you for your + MySQL root password then attempts to do everything else + automatically using values from myrpki.conf; or + + b) You can do it manually. + + The first approach is simple: + + $ python setup-sql.py + Please enter your MySQL root password: + + The script should tell you what databases it creates. You can use + the -v option if you want to see more details about what it's doing. + + If you'd prefer to do the SQL setup manually, perhaps because you + have valuable data in other MySQL databases and you don't want to + trust some random setup script with your MySQL root password, + you'll need to use the MySQL command line tool, as follows: + + $ mysql -u root -p + + mysql> CREATE DATABASE irdb_database; + mysql> GRANT all ON irdb_database.* TO irdb_user@localhost IDENTIFIED BY 'irdb_password'; + mysql> USE irdb_database; + mysql> SOURCE $top/rpkid/irdbd.sql; + mysql> CREATE DATABASE rpki_database; + mysql> GRANT all ON rpki_database.* TO rpki_user@localhost IDENTIFIED BY 'rpki_password'; + mysql> USE rpki_database; + mysql> SOURCE $top/rpkid/rpkid.sql; + mysql> COMMIT; + mysql> quit + + where "irdb_database", "irdb_user", "irdb_password", + "rpki_database", "rpki_user", and "rpki_password" are the + appropriate values from your configuration file. + + If you are running pubd and doing manual SQL setup, you'll also + have to do: + + $ mysql -u root -p + mysql> CREATE DATABASE pubd_database; + mysql> GRANT all ON pubd_database.* TO pubd_user@localhost IDENTIFIED BY 'pubd_password'; + mysql> USE pubd_database; + mysql> SOURCE $top/rpkid/pubd.sql; + mysql> COMMIT; + mysql> quit + +6) Run myirbe.py -b to set up the initial BPKI structure needed to run + your daemons: + + $ python $top/myrpki/myirbe.py -b + + The -b option tells myrpki.py that you want it to stop after the + initial BPKI setup, regardless of whether it thinks this is + necessary. If you have not done this before it should tell you + that it has updated the BPKI and that you need to (re)start daemons + now. + +7) If you are running your own publication repository (that is, if you + are running pubd), you will also need to set up an rsyncd server or + configure your existing one to serve pubd's output. There's a + sample configuration file in $top/myrpki/examples/rsyncd.conf, but + you may need to do something more complicated if you are already + running rsyncd for other purposes. See the rsync(1) and + rsyncd.conf(5) manual pages for more details. + +8) Start the daemons. You can use $top/myrpki/start-servers.py to do + this, or write your own script. + + If you intend to run pubd, you should make sure that the directory + you specified as publication-base in the [pubd] section exists and + is writable by the userid that will be running pubd, and should + also make sure to start rsyncd. + +9) Run myirbe.py again, twice, this time with no arguments. + + $ python $top/myrpki/myirbe.py + $ python $top/myrpki/myirbe.py + + The reason for running myirbe.py twice at this point is explained + in the Introduction section, above; in brief, the first run sets up + almost everything, but a second pass is required to generate the + BSC certificate. + +At this point, if everything went well, rpkid should be up, +configured, and starting to obtain resource certificates from its +parents, generate CRLs and manifests, and so forth. At this point you +should go figure out how to use the relying party tool, rcynic: see +$top/rcynic/README if you haven't already done so. + +If and when you change your CSV files, you should run myirbe.py again +to feed the changes into the daemons. + +GETTING STARTED -- HOSTING CASE + +If you are running rpkid not just for your own resources but also to +host other resource holders (see "HOSTED CASE" above), your setup will +be almost the same as in the self-hosted case (see "SELF-HOSTED CASE", +above), with one procedural change: you will need to tell myirbe.py to +process the XML files produced by the resource holders you are +hosting. You do this by specifying the names of all those XML files +on myirbe's command line. So, if you are hosting two friends, Alice +and Bob, then, everywhere the instructions for the self-hosted case +say to run myirbe.py with no arguments, you will instead run it with +the names of Alice's and Bob's XML files: + + $ python $top/myrpki/myirbe.py alice.xml bob.xml + +Note that myirbe.py sometimes modifies these XML files, in which case +it will write them back to the same filenames. While it is possible +to figure out the set of circumstances in which myirbe.py will modify +XML files (at present, this only happens when myirbe.py has to ask +rpkid to create a new BSC keypair and PKCS #10 certificate request), +it may be easiest just to ship back an updated copy of the XML file +after every you run myirbe.py. + +GETTING STARTED -- "PURE" HOSTING CASE + +In general we assume that anybody who bothers to run rpkid is also a +resource holder, but the software does not insist on this. If you are +running rpkid solely for others and have no resources of your own, the +process is almost identical to the "HOSTING CASE", above. The one +change is that you should *not* have a [myrpki] section in your +configuration file. + +A (perhaps) slightly-more-plausible use for this capability would be +if you are an rpkid-running resource holder who wants for some reason +to keep the resource-holding side of your operation completely +separate from the rpkid-running side of your operation. This is +essentially the pure-hosting model, just with an internal hosted +entity within a different part of your own organization. + +DATA YOU NEED FROM YOUR RPKI PARENT AND PUBLICATION SERVICE + +In order to connect to your RPKI parent, you will need to supply your +BPKI trust anchor to your parent and obtain four pieces of data from +your parent. + +Assuming that you are using something resembling the default +configuration, your BPKI trust anchor will be bpki.myrpki/ca.cer. +This is an OpenSSL "PEM" format file. You will need to provide this +to your RPKI parent. + +The data you need from your parent are: + +- The service URL for your entry point into your parent's rpkid. + Typically this will be a URL of the form: + + https://example.org:port/up-down/parenthandle/myhandle + + where "example.org" and "port" are the DNS name and TCP port of your + parent's rpkid service, "parenthandle" is your parent's name + (handle) for itself, and "myhandle" is your parent's name (handle) + for you; + +- Your parent's BPKI trust anchor for its resource-holding persona + (the entity represented by "parenthandle", above); + +- Your parent's BPKI trust anchor for daemons it operates; and + +- The handle by which your parent refers to you in its database, + generally the same as "myhandle" in the service URL. + +The need for two separate BPKI trust anchors for your parent is due to +a limitation of the HTTPS protocol; recent extensions to TLS provide a +way to work around this limitation, but at this point in time rpkid +can't assume support for the TLS extension in question. Roughly +speaking, the first BPKI trust anchor corresponds to the your parent +as a resource-holding entity, while the second corresponds to your +parent as an rpkid-operating entity. + +These four data correspond, in order, to the second, third, fourth, +and fifth columns in your parents.csv file. In most cases you will +have only one parent, so there will be only one line in that file. + +The first field in the parents.csv file is your name for your parent, +which can be any name you like so long as it doesn't conflict with +your name for another parent. + +The sixth field in the parents.csv file determines the base rsync URI +for objects signed by certificates issued by this parent. If you are +using an external publication service (recommended), your parent must +supply this URI as well; a typical value would be +rsync://example.org/Dad/Me/ or rsync://example.org/Grandma/Dad/Me/. + +If you are running your own copy of pubd, this URI should point to the +directory that corresponds to the publication-base setting in the +[pubd] section of your configuration file. + +If you are using an external publication service (which might be your +parent, grandparent, or any ancestor all the way up to the root), your +publication service will also need to tell you: + +- The service URL for the publication service (pubd_base parameter in + [myirbe] section of your configuration file); + +- The publication service's name for you (repository_handle field in + [myrpki] section of your configuration file); and + +- The BPKI trust anchor for the publication service + (repository_bpki_certificate field in [myrpki] section of your + configuration file). + +Note that the first of these three parameters only applies if you are +running rpkid, while the second and third apply even if your resources +are hosted on somebody else's rpkid. In effect, this means that all +the entities sharing a single rpkid must also share a single +publication service. This is a restriction of the myrpki/myirbe +software, not rpkid itself, so it could be removed if there were a +strong need to do so, but given that each additional publication +service imposes a small additional burden on every relying party in +the world, we do not view this restriction as a problem. + +DATA YOU NEED TO GIVE YOUR RPKI CHILDREN AND USERS OF YOUR PUBLICATION SERVICE + +First, read the previous section describing what children and +publication clients expect to receive. + +- The service URL for your rpkid will be an HTTPS URL of the form + + https://example.org:port/up-down/yourhandle/childhandle + + where "example.org" and "port" are the DNS name and TCP port of your + rpkid service ([rpkid] section of your configuration file), + "yourhandle" is the handle parameter from the [myrpki] section of + your configuration file, and "childhandle" is this child's handle as + it appears in the first columns of your children.csv, asns.csv, and + prefixes.csv files; + +- The BPKI trust anchor for your resource-holding persona is your + bpki.myrpki/ca.cer; + +- The BPKI trust anchor for daemons you operate is your + bpki.myirbe/ca.cer; and + +- The handle by which you refer to your child is the same as + "childhandle", above. + +If you are operating a publication service, you will also need to +supply: + +- Your pubd service URL, which will be an HTTPS URL of the form + + https://example.org:port/ + + where "example.org" and "port" are the server-host and server-port + parameters from the [pubd] section of your configuration file; + +- Your name for this publication client, which is the first column of + your pubclients.csv file (note that this can be a structured name + using "/" characters as a hierarchy delimiter); and + +- The BPKI trust anchor for the daemons you operate + (bpki.myirbe/ca.cer). + +Note that, if you are operating pubd, it's best for relying parties if +your children's publication points are underneath yours within the +publication hierarchy, to allow rsync to check for updates as +efficiently as possible. pubd's support for hierarchical client +handles is intended to simplify this: if you have a child Alice, who +has children Bob and Bill, and you, your children, and your +grandchildren will all be using your publication service, you might +assign <client_handle> and <sia_base> parameters (first and third +fields in pubclients.csv) as follows: + +Me rsync://rpki.example.org/Me/ +Me/Alice rsync://rpki.example.org/Me/Alice/ +Me/Alice/Bob rsync://rpki.example.org/Me/Alice/Bob/ +Me/Alice/Bill rsync://rpki.example.org/Me/Alice/Bill/ + +Note that you will need trust anchors for your children and any +publication clients. In both cases the trust anchor you need is the +child's or client's resource-holding BPKI trust anchor +(bpki.myrpki/ca.cer); who operates the rpkid that host your children +or publication clients is not strictly relevant to the authorization +model, what matters is who holds the resources and is authorized to +request and publish RPKI data derived from them. + +TROUBLESHOOTING + +If you run into trouble setting up this package, the first thing to do +is categorize the kind of trouble you are having. If you've gotten +far enough to be running the daemons, check their log files. If +you're seeing Python exceptions, read the error messages. If you're +getting TLS errors, check to make sure that you're using all the right +BPKI certificates and service contact URLs. + +TLS configuration errors are, unfortunately, notoriously difficult to +debug, because connections due to misconfiguration usually fail early, +deep in the guts of the OpenSSL TLS code, where there isn't enough +application context available to provide useful error messages. + +If you've completed the steps above, everything appears to have gone +OK, but nothing seems to be happening, the first thing to do is check +the logs to confirm that nothing is actively broken. rpkid's log +should include messages telling you when it starts and finishes its +internal "cron" cycle. It can take several cron cycles for resources +to work their way down from your parent into a full set of +certificates and ROAs, so have a little patience. rpkid's log should +also include messages showing every time it contacts its parent(s) or +attempts to publish anything. + +rcynic in fully verbose mode provides a fairly detailed explanation of +what it's doing and why objects that fail have failed. + +You can use rsync (sic) to examine the contents of a publication +repository one directory at a time, without attempting validation, by +running rsync with just the URI of the directory on its command line: + + $ rsync rsync://rpki.example.org/where/ever/ + +[Maybe there should be something here explaining how to use +irbe_cli.py for debugging, but the syntax is fairly obscure as it's +just a command line interface to the left-right and publication +protocols -- almost certainly want a friendlier tool for +troubleshooting.] + +KNOWN ISSUES + +The lxml package provides a Python interface to the Gnome libxml2 and +libxslt C libraries. This code has been quite stable for several +years, but initial testing with lxml compiled and linked against a +newer version of libxml2 ran into problems (specifically, gratuitous +RelaxNG schema validation failures). libxml2 2.7.3 worked; libxml2 +2.7.5 did not work on the test machine in question. Reverting to +libxml2 2.7.3 fixed the problem. Rewriting the two lines of Python +code that were triggering the lxml bug appears to have solved the +problem, so the code now works properly with libxml 2.7.5, but if you +start seeing weird XML validation failures, it might be another +variation of this lxml bug. + +An earlier version of this code ran into problems with what appears to +be an implementation restriction in the the GNU linker ("ld") on +64-bit hardware, resulting in obscure build failures. The workaround +for this required use of shared libraries and is somewhat less +portable than the original code, but without it the code simply would +not build in 64-bit environments with the GNU tools. The current +workaround appears to behave properly, but the workaround requires +that the pathname to the RFC-3779-aware OpenSSL shared libraries be +built into the _POW.so Python extension module. At the moment, in the +absence of "make install" targets for the Python code and libraries, +this means the build directory; eventually, once we're using autoconf +and installation targets, this will be the installation directory. If +necessary, you can override this by setting the LD_LIBRARY_PATH +environment variable, see the ld.so man page for details. This is a +relatively minor variation on the usual build issues for shared +libraries, it's just annoying because shared libraries should not be +needed here and would not be if not for this GNU linker issue. + + + +Sketch towards a simple description of the BPKI (sic). + +This started out as notes to myself during a redesign, and badly needs +rewriting. + +Hosted (myrpki) entity needs: + +- Self-signed BPKI root (doesn't really need to be self-signed, nobody + else will care, but self-signed is simplest for our purposes). This + is what we've been calling the "self" cert in testbed.py. + +- BSC EE issued by self-signed root. + +- Cross-certs of every foreign entity (parent, child, or pubd): these + are CA certs with pathLenConstraint 0. Input for this cross-cert is + self-signed (or whatever) from foreign entity, output is + pathLenConstraint 0 CA cert issued by myrpki entity's own + self-signed root. + +Hosting rpkid (myirbe) needs: + +- Self-signed BPKI root + +- BSC EE certs for rpkid, irdbd, irbe_cli, etc + +- For each hosted entity (including self-hosting): + + Cross-cert of hosted entity's root, issued by rpkid root: CA cert + with pathLenConstraint 1 + + In theory that's all that's required, everything else is handled + through the hosted entity's cert chain. + +pubd needs: + +- Self signed root (might share with rpkid but let's keep it separate + conceptually) + +- BSC EE certs for pubd and irbe_cli + +- For each client entity of pubd: + + Cross-cert of client entity's self cert (pathLenConstraint 0). + + This should allow pubd to verify clients' BSC EE certs without + getting into transitive CA relationships. + +rootd (when applicable at all) needs: + +- Self-signed root + +- BSC EE cert for talking up-down (server) with one and only child + +- Cross-cert (pathLenConstraint 0) of one and only child's self cert. diff --git a/myrpki.rototill/arin-rootcert.py b/myrpki.rototill/arin-rootcert.py new file mode 100644 index 00000000..09180af6 --- /dev/null +++ b/myrpki.rototill/arin-rootcert.py @@ -0,0 +1,69 @@ +""" +Generate config for a test RPKI root certificate for resources +specified in asns.csv and prefixes.csv. + +This script is separate from arin-to-csv.py so that we can convert on +the fly rather than having to pull the entire database into memory. + +$Id$ + +Copyright (C) 2009 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import csv, myrpki, sys + +holder = "arin" + +if len(sys.argv) == 2: + holder = sys.argv[1] +elif len(sys.argv) > 1: + raise RuntimeError, "Usage: %s [holder]" % sys.argv[0] + +print '''\ +[req] +default_bits = 2048 +default_md = sha256 +distinguished_name = req_dn +prompt = no +encrypt_key = no + +[req_dn] +CN = Pseudo-%(HOLDER)s testbed root RPKI certificate + +[x509v3_extensions] +basicConstraints = critical,CA:true +subjectKeyIdentifier = hash +keyUsage = critical,keyCertSign,cRLSign +subjectInfoAccess = 1.3.6.1.5.5.7.48.5;URI:rsync://%(holder)s.rpki.net/%(holder)s/,1.3.6.1.5.5.7.48.10;URI:rsync://%(holder)s.rpki.net/%(holder)s/root.mnf +certificatePolicies = critical,1.3.6.1.5.5.7.14.2 +sbgp-autonomousSysNum = critical,@rfc3779_asns +sbgp-ipAddrBlock = critical,@rfc3997_addrs + +[rfc3779_asns] +''' % { "holder" : holder.lower(), + "HOLDER" : holder.upper() } + +for i, asn in enumerate(asn for handle, asn in myrpki.csv_open("asns.csv")): + print "AS.%d = %s" % (i, asn) + +print '''\ + +[rfc3997_addrs] + +''' + +for i, prefix in enumerate(prefix for handle, prefix in myrpki.csv_open("prefixes.csv")): + v = 6 if ":" in prefix else 4 + print "IPv%d.%d = %s" % (v, i, prefix) diff --git a/myrpki.rototill/arin-to-csv.py b/myrpki.rototill/arin-to-csv.py new file mode 100644 index 00000000..fc98bb64 --- /dev/null +++ b/myrpki.rototill/arin-to-csv.py @@ -0,0 +1,119 @@ +""" +Parse a WHOIS research dump and write out (just) the RPKI-relevant +fields in myrpki-format CSV syntax. + +NB: The input data for this script comes from ARIN under an agreement +that allows research use but forbids redistribution, so if you think +you need a copy of the data, please talk to ARIN about it, not us. + +$Id$ + +Copyright (C) 2009 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import gzip, csv, myrpki + +class Handle(object): + + want_tags = () + + debug = False + + def set(self, tag, val): + if tag in self.want_tags: + setattr(self, tag, "".join(val.split(" "))) + + def check(self): + for tag in self.want_tags: + if not hasattr(self, tag): + return False + if self.debug: + print repr(self) + return True + +class ASHandle(Handle): + + want_tags = ("ASHandle", "ASNumber", "OrgID") + + def __repr__(self): + return "<%s %s.%s %s>" % (self.__class__.__name__, + self.OrgID, self.ASHandle, self.ASNumber) + + def finish(self, ctx): + if self.check(): + ctx.asns.writerow((self.OrgID, self.ASNumber)) + +class NetHandle(Handle): + + NetType = None + + want_tags = ("NetHandle", "NetRange", "NetType", "OrgID") + + def finish(self, ctx): + if self.NetType in ("allocation", "assignment") and self.check(): + ctx.prefixes.writerow((self.OrgID, self.NetRange)) + + def __repr__(self): + return "<%s %s.%s %s %s>" % (self.__class__.__name__, + self.OrgID, self.NetHandle, + self.NetType, self.NetRange) + +class V6NetHandle(NetHandle): + + want_tags = ("V6NetHandle", "NetRange", "NetType", "OrgID") + + def __repr__(self): + return "<%s %s.%s %s %s>" % (self.__class__.__name__, + self.OrgID, self.V6NetHandle, + self.NetType, self.NetRange) + +class main(object): + + types = { + "ASHandle" : ASHandle, + "NetHandle" : NetHandle, + "V6NetHandle" : V6NetHandle } + + @staticmethod + def parseline(line): + tag, sep, val = line.partition(":") + assert sep, "Couldn't find separator in %r" % line + return tag.strip(), val.strip() + + @staticmethod + def csvout(fn): + return csv.writer(open(fn, "w"), dialect = myrpki.csv_dialect) + + def __init__(self): + self.asns = self.csvout("asns.csv") + self.prefixes = self.csvout("prefixes.csv") + f = gzip.open("arin_db.txt.gz") + cur = None + for line in f: + line = line.expandtabs().strip() + if not line: + if cur: + cur.finish(self) + cur = None + elif not line.startswith("#"): + tag, val = self.parseline(line) + if cur is None: + cur = self.types[tag]() if tag in self.types else False + if cur: + cur.set(tag, val) + if cur: + cur.finish(self) + +main() diff --git a/myrpki.rototill/cherrypy-example.py b/myrpki.rototill/cherrypy-example.py new file mode 100644 index 00000000..c5c97fef --- /dev/null +++ b/myrpki.rototill/cherrypy-example.py @@ -0,0 +1,12 @@ +# $Id$ + +import cherrypy + +class HelloWorld(object): + + @cherrypy.expose + def index(self): + return "Hello world!" + +if __name__ == "__main__": + cherrypy.quickstart(HelloWorld()) diff --git a/myrpki.rototill/children-to-pubclients.py b/myrpki.rototill/children-to-pubclients.py new file mode 100644 index 00000000..025d3d42 --- /dev/null +++ b/myrpki.rototill/children-to-pubclients.py @@ -0,0 +1,42 @@ +""" +Convert children.csv to (initial) pubclients.csv. You may wish to +play sort/join/etc games with the output of this to avoid overwriting +other publication clients you've configured. + +$Id$ + +Copyright (C) 2009 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import sys, csv, myrpki, getopt, time, os, rpki.config + +os.environ["TZ"] = "UTC" +time.tzset() + +cfg_file = "myrpki.conf" + +opts, argv = getopt.getopt(sys.argv[1:], "c:h?", ["config=", "help"]) +for o, a in opts: + if o in ("-h", "--help", "-?"): + print __doc__ + sys.exit(0) + if o in ("-c", "--config"): + cfg_file = a + +base = rpki.config.parser(cfg_file, "myirbe").get("rsync_base") + +csv.writer(sys.stdout, dialect = myrpki.csv_dialect).writerows( + (handle, cert, "%s/children/%s/" % (base.rstrip("/"), handle)) + for handle, expiration, cert in myrpki.csv_open("children.csv")) diff --git a/myrpki.rototill/examples/asns.csv b/myrpki.rototill/examples/asns.csv new file mode 100644 index 00000000..804cf839 --- /dev/null +++ b/myrpki.rototill/examples/asns.csv @@ -0,0 +1,8 @@ +# $Id$ +# +# Syntax: <child_handle> <asn> +# +# NB: Comment lines are not allowed in these files, this one is only +# present to explain the example +# +Alice 64533 diff --git a/myrpki.rototill/examples/children.csv b/myrpki.rototill/examples/children.csv new file mode 100644 index 00000000..da29e8b5 --- /dev/null +++ b/myrpki.rototill/examples/children.csv @@ -0,0 +1,9 @@ +# $Id$ +# +# Syntax: <child_handle> <validitydate> <bpki_cert_filename> +# +# NB: Comment lines are not allowed in these files, this one is only +# present to explain the example +# +Alice 2009-07-27T08:24:53Z Alice.ta.cer +Bob 2009-07-27T08:24:53Z Bob.ta.cer diff --git a/myrpki.rototill/examples/myrpki.conf b/myrpki.rototill/examples/myrpki.conf new file mode 100644 index 00000000..0eded59b --- /dev/null +++ b/myrpki.rototill/examples/myrpki.conf @@ -0,0 +1,411 @@ +# $Id: myrpki.conf 2722 2009-08-31 22:24:48Z sra $ +# +# Config file for myrpki.py, myirbe.py, and RPKI daemons when used +# with myrpki.py etc. Notes: +# +# - There's some duplication of settings between some of the sections, +# because each of the several daemons and control programs was +# written as a free-standing program. Lumping all of the config for +# all of them into a single config file is just a convenience for +# simple configurations; in complex cases you might not have any two +# of them running on the same machine. +# +# - This config file is also read by the OpenSSL command line tool +# running under mypki.py, so syntax must remain compatable with both +# OpenSSL and Python config file parsers, and there's a big chunk of +# OpenSSL voodoo towards the end of this file. + +################################################################ + +[myrpki] + +# Handle naming hosted resource-holding entity (<self/>) represented +# by this myrpki instance. Syntax is an identifier (ASCII letters, +# digits, hyphen, underscore -- no whitespace, non-ASCII characters, +# or other punctuation). You need to set this. + +handle = Me + +# BPKI trust anchor for the repository in which this <self/> will be +# publishing its outputs. You need to set this. + +repository_bpki_certificate = repository-ta.cer + +# Name by which repository will know this <self/>. This may be a +# structured handle, eg, "Grandma/Mom/Me" or might be a simple handle, +# depending on how the repository is set up. Syntax is same as +# "handle", with the addition of "/" characters as an allowed +# delimiter. You need to set this. + +repository_handle = Me + +# Names of various input and output files. Don't change these without +# a good reason. + +roa_csv = roas.csv +children_csv = children.csv +parents_csv = parents.csv +prefix_csv = prefixes.csv +asn_csv = asns.csv +xml_filename = myrpki.xml +bpki_directory = bpki.myrpki + +################################################################# + +[myirbe] + +# Base of service URL for pubd. myirbe.py uses this value to +# configure <repository/> objects in rpkid. If you are running your +# own copy of pubd (see "want_pubd"), myirbe.py also uses this to +# contact your copy of pubd in order to configure it. +# +# You need to configure this. + +pubd_base = https://pubd.example.org:4402/ + +# Base of service URL for rpkid. myirbe.py uses this to contact your +# rpkid so it can configure it. +# +# You need to configure this. + +rpkid_base = https://rpkid.example.org:4404 + +# Whether you want myirbe.py to attempt to configure your own copy of +# pubd. In general, it's best to use your parent's pubd if you can, +# to reduce the overall number of publication sites that relying +# parties need to check, so don't enable this unless you have a good +# reason. See the [pubd] section if you do enable this. +# +# Enabling this when you are -not- running your own copy of pubd will +# cause myirbe.py to fail when it attempts to perform runtime +# configuration of your nonexistant pubd. + +want_pubd = false + +# Whether you want myirbe.py to generate BPKI certs for running your +# very own copy of rootd. Don't enable this unless you really know +# what you're doing. See [rootd] section below for further comments. + +want_rootd = false + +# Where to put BPKI stuff for the IRBE operator (entity that operates +# rpkid etc). Don't change this without a reason. + +bpki_directory = bpki.myirbe + +################################################################# + +[rpkid] + +# MySQL database name, user name, and password for rpkid to use to +# store its data. You need to configure these. + +sql-database = rpki +sql-username = rpki +sql-password = fnord + +# Host and port on which rpkid should listen for HTTPS service +# requests. These should match rpkid_base in the [myirbe] section. +# You need to configure these. + +server-host = rpkid.example.org +server-port = 4404 + +# HTTPS service URL rpkid should use to contact irdbd. If irdbd is +# running on the same machine as rpkid, this can and probably should +# be a loopback URL, since nobody but rpkid needs to talk to irdbd. + +irdb-url = https://localhost:4403/ + +# Where rpkid should look for BPKI certs and keys used in the +# left-right protocol. The following values match where myirbe.py +# will have placed things. Don't change these without a reason. + +bpki-ta = bpki.myirbe/ca.cer +rpkid-key = bpki.myirbe/rpkid.key +rpkid-cert = bpki.myirbe/rpkid.cer +irdb-cert = bpki.myirbe/irdbd.cer +irbe-cert = bpki.myirbe/irbe.cer + +################################################################# + +[irdbd] + +# MySQL database name, user name, and password for irdbd to use to +# store its data. You need to configure these. + +sql-database = irdb +sql-username = irdb +sql-password = fnord + +# HTTP service URL irdbd should listen on. This should match the +# irdb-url parameter in the [rpkid] section; see comments there. + +https-url = https://localhost:4403/ + +# Where irdbd should look for BPKI certs and keys used in the +# left-right protocol. The following values match where myirbe.py +# will have placed things. Don't change these without a reason. + +bpki-ta = bpki.myirbe/ca.cer +rpkid-cert = bpki.myirbe/rpkid.cer +irdbd-cert = bpki.myirbe/irdbd.cer +irdbd-key = bpki.myirbe/irdbd.key + +################################################################# + +[pubd] + +# MySQL database name, user name, and password for pubd to use to +# store (some of) its data. You need to configure these. + +sql-database = pubd +sql-username = pubd +sql-password = fnord + +# Root of directory tree where pubd should write out published data. +# You need to configure this, and the configuration should match up +# with the directory where you point rsyncd. Neither pubd nor rsyncd +# much cares -where- you tell them to put this stuff, the important +# thing is that the rsync:// URIs in generated certificates match up +# with the published objects so that relying parties can find and +# verify rpkid's published outputs. + +publication-base = publication/ + +# Host and port on which pubd should listen for HTTPS service +# requests. These should match pubd_base in the [myirbe] section. +# You need to configure these. + +server-host = pubd.example.org +server-port = 4402 + +# Where pubd should look for BPKI certs and keys used in the +# left-right protocol. The following values match where myirbe.py +# will have placed things. Don't change these without a reason. + +bpki-ta = bpki.myirbe/ca.cer +pubd-cert = bpki.myirbe/pubd.cer +pubd-key = bpki.myirbe/pubd.key +irbe-cert = bpki.myirbe/irbe.cer + +################################################################# + +[irbe_cli] + +# HTTPS service URL for rpkid + +rpkid-url = https://rpkid.example.org:4404/left-right/ + +# BPKI certificates and keys for talking to rpkid + +rpkid-bpki-ta = bpki.myirbe/ca.cer +rpkid-irbe-key = bpki.myirbe/irbe.key +rpkid-irbe-cert = bpki.myirbe/irbe.cer +rpkid-cert = bpki.myirbe/rpkid.cer + +# HTTPS service URL for pubd + +pubd-url = https://localhost:4402/control/ + +# BPKI certificates and keys for talking to pubd + +pubd-bpki-ta = bpki.myirbe/ca.cer +pubd-irbe-key = bpki.myirbe/irbe.key +pubd-irbe-cert = bpki.myirbe/irbe.cer +pubd-cert = bpki.myirbe/pubd.cer + +################################################################# + +# You don't need to run rootd unless you're IANA, are certifying +# private address space, or are an RIR which refuses to accept IANA as +# the root of the public address hierarchy. +# +# Ok, if that wasn't enough to scare you off: rootd is a kludge, and +# needs to be rewritten, or, better, merged into rpkid. It does a +# number of things wrong, and requires far too many configuration +# parameters. You have been warned.... + +[rootd] + +# BPKI certificates and keys for rootd + +bpki-ta = bpki.myirbe/ca.cer +rootd-bpki-crl = bpki.myirbe/ca.crl +rootd-bpki-cert = bpki.myirbe/rootd.cer +rootd-bpki-key = bpki.myirbe/rootd.key +child-bpki-cert = bpki.myirbe/child.cer + +# Server port on which rootd should listen. + +server-port = 4401 + +# Where rootd should write its output. Yes, rootd should be using +# pubd instead of publishing directly, but it doesn't. + +rpki-root-dir = publication/ + +# rsync URI for directory containing rootd's outputs + +rpki-base-uri = rsync://rpki.example.org/Me/ + +# rsync URI for rootd's root (self-signed) RPKI certificate + +rpki-root-cert-uri = rsync://rpki.example.org/Me/root.cer + +# Private key corresponding to rootd's root RPKI certificate + +rpki-root-key = bpki.myirbe/ca.key + +# Filename (as opposed to rsync URI) of rootd's root RPKI certificate + +rpki-root-cert = publication/root.cer + +# Where rootd should stash a copy of the PKCS #10 request it gets from +# its one (and only) child + +rpki-subject-pkcs10 = rootd.subject.pkcs10 + +# Lifetime of the one and only certificate rootd issues + +rpki-subject-lifetime = 30d + +# Filename (relative to rootd-base-uri and rpki-root-dir) of the CRL +# for rootd's root RPKI certificate + +rpki-root-crl = root.crl + +# Filename (relative to rootd-base-uri and rpki-root-dir) of the +# manifest for rootd's root RPKI certificate + +rpki-root-manifest = root.mnf + +# Up-down protocol class name for RPKI certificate rootd issues to its +# one (and only) child + +rpki-class-name = Me + +# Filename (relative to rootd-base-uri and rpki-root-dir) of the one +# (and only) RPKI certificate rootd issues + +rpki-subject-cert = Me.cer + +# The last four paramters in this section are really parameters for +# myirbe.py to use when constructing rootd's root RPKI certificate, +# via an indirection hack in the OpenSSL voodoo portion of this file. +# Don't ask why some of these are duplicated from other paramters in +# this section, you don't want to know (really, you don't). + +# ASNs to include in rootd's root RPKI certificate, in openssl.conf format + +root_cert_asns = AS:0-4294967295 + +# IP addresses to include in rootd's root RPKI certificate, in +# openssl.conf format + +root_cert_addrs = IPv4:0.0.0.0/0,IPv6:0::/0 + +# Whatever you put in rpki-base-uri, earlier in this section + +root_cert_sia = rsync://rpki.example.org/Me/ + +# root_cert_sia + rpki-root-manifest + +root_cert_manifest = rsync://rpki.example.org/Me/root.mnf + +################################################################# + +# Constants for OpenSSL voodoo portion of this file, to make them +# easier to find. + +[constants] + +# Digest algorithm. Don't change this. + +digest = sha256 + +# RSA key length. Don't change this. + +key_length = 2048 + +# Lifetime of BPKI certificates (and rootd RPKI root certificate). +# Don't change this unless you know what you're doing. + +cert_days = 365 + +# Lifetime of BPKI CRLs. Don't change this unless you know what +# you're doing. + +crl_days = 365 + +################################################################# + +# The rest of this file is OpenSSL configuration voodoo. Don't touch +# anything below here even if you -do- know what you're doing. Even +# by OpenSSL standards, some of this is weird, and interacts in +# non-obvious ways with code in myrpki.py and myirbe.py. If you touch +# this stuff and something breaks, don't say you weren't warned. + +[req] +default_bits = ${constants::key_length} +default_md = ${constants::digest} +distinguished_name = req_dn +prompt = no +encrypt_key = no + +[req_dn] +CN = Dummy name for certificate request + +[ca_x509_ext_ee] +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always + +[ca_x509_ext_xcert0] +basicConstraints = critical,CA:true,pathlen:0 +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always + +[ca_x509_ext_xcert1] +basicConstraints = critical,CA:true,pathlen:1 +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always + +[ca_x509_ext_ca] +basicConstraints = critical,CA:true +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always + +[ca] +default_ca = ca +dir = ${ENV::BPKI_DIRECTORY} +new_certs_dir = $dir +database = $dir/index +certificate = $dir/ca.cer +private_key = $dir/ca.key +default_days = ${constants::cert_days} +default_crl_days = ${constants::crl_days} +default_md = ${constants::digest} +policy = ca_dn_policy +unique_subject = no +serial = $dir/serial +crlnumber = $dir/crl_number + +[ca_dn_policy] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional +givenName = optional +surname = optional + +[rootd_x509_extensions] +basicConstraints = critical,CA:true +subjectKeyIdentifier = hash +keyUsage = critical,keyCertSign,cRLSign +subjectInfoAccess = 1.3.6.1.5.5.7.48.5;URI:${rootd::root_cert_sia},1.3.6.1.5.5.7.48.10;URI:${rootd::root_cert_manifest} +sbgp-autonomousSysNum = critical,${rootd::root_cert_asns} +sbgp-ipAddrBlock = critical,${rootd::root_cert_addrs} +certificatePolicies = critical,1.3.6.1.5.5.7.14.2 diff --git a/myrpki.rototill/examples/parents.csv b/myrpki.rototill/examples/parents.csv new file mode 100644 index 00000000..f92eddeb --- /dev/null +++ b/myrpki.rototill/examples/parents.csv @@ -0,0 +1,8 @@ +# $Id$ +# +# Syntax: <parent_handle> <service_uri> <cms_bpki_cert_filename> <https_bpki_cert_filename> <myhandle> <sia_base> +# +# NB: Comment lines are not allowed in these files, this one is only +# present to explain the example +# +Mom https://localhost:4414/up-down/Mom/Becca Mom.ta.cer Mom.rpkid.cer Becca rsync://rpki.example.org/Me/ diff --git a/myrpki.rototill/examples/prefixes.csv b/myrpki.rototill/examples/prefixes.csv new file mode 100644 index 00000000..160f9339 --- /dev/null +++ b/myrpki.rototill/examples/prefixes.csv @@ -0,0 +1,11 @@ +# $Id$ +# +# Syntax: <child_handle> <prefix>/<length> +# or: <child_handle> <min>-<max> +# +# NB: Comment lines are not allowed in these files, this one is only +# present to explain the example +# +Alice 192.0.2.0/27 +Bob 192.0.2.44-192.0.2.100 +Bob 10.0.0.0/8 diff --git a/myrpki.rototill/examples/pubclients.csv b/myrpki.rototill/examples/pubclients.csv new file mode 100644 index 00000000..6336a1a6 --- /dev/null +++ b/myrpki.rototill/examples/pubclients.csv @@ -0,0 +1,10 @@ +# $Id$ +# +# Syntax: <client_handle> <bpki_cert_filename> <sia_base> +# +# NB: Comment lines are not allowed in these files, this one is only +# present to explain the example +# +Me bpki.myrpki/ca.cer rsync://rpki.example.org/Me/ +Me/Alice pubd-client-certs/Alice.cer rsync://rpki.example.org/Me/Alice/ +Me/Bob pubd-client-certs/Bob.cer rsync://rpki.example.org/Me/Bob/ diff --git a/myrpki.rototill/examples/roas.csv b/myrpki.rototill/examples/roas.csv new file mode 100644 index 00000000..4343ada0 --- /dev/null +++ b/myrpki.rototill/examples/roas.csv @@ -0,0 +1,8 @@ +# $Id$ +# +# Syntax: <prefix>/<length>-<maxlength> <asn> <group> +# +# NB: Comment lines are not allowed in these files, this one is only +# present to explain the example +# +10.3.0.44/32 666 Mom diff --git a/myrpki.rototill/examples/rsyncd.conf b/myrpki.rototill/examples/rsyncd.conf new file mode 100644 index 00000000..d0a9cd97 --- /dev/null +++ b/myrpki.rototill/examples/rsyncd.conf @@ -0,0 +1,30 @@ +# $Id$ +# +# Sample rsyncd.conf file for use with pubd. You may need to +# customize this for the conventions on your system. See the rsync +# and rsyncd.conf manual pages for a complete explanation of how to +# configure rsyncd, this is just a simple configuration to get you +# started. +# +# There are two parameters in the following which you should set to +# appropriate values for your system: +# +# "myname" is the rsync module name to configure, as in +# "rsync://rpki.example.org/myname/" +# +# "/some/where/publication" is the absolute pathname of the directory +# where you told pubd to place its outputs (see the publication_base +# parameter in the [pubd] section of myrpki.conf) +# +# You may need to adjust other parameters for your system environment. + +pid file = /var/run/rsyncd.pid +uid = nobody +gid = nobody + +[myname] + use chroot = no + read only = yes + transfer logging = yes + path = /some/where/publication + comment = RPKI Testbed diff --git a/myrpki.rototill/myirbe.py b/myrpki.rototill/myirbe.py new file mode 100644 index 00000000..ad54c9aa --- /dev/null +++ b/myrpki.rototill/myirbe.py @@ -0,0 +1,549 @@ +""" +IRBE-side stuff for myrpki tools. + +The basic model here is that each entity with resources to certify +runs the myrpki tool, but not all of them necessarily run their own +RPKi engines. The entities that do run RPKI engines get data from the +entities they host via the XML files output by the myrpki tool. Those +XML files are the input to this script, which uses them to do all the +work of constructing certificates, populating SQL databases, and so +forth. A few operations (eg, BSC construction) generate data which +has to be shipped back to the resource holder, which we do by updating +the same XML file. + +In essence, the XML files are a sneakernet (or email, or carrier +pigeon) communication channel between the resource holders and the +RPKI engine operators. + +As a convenience, for the normal case where the RPKI engine operator +is itself a resource holder, this script also runs the myrpki script +directly to process the RPKI engine operator's own resources. + +Note that, due to the back and forth nature of some of these +operations, it may take several cycles for data structures to stablize +and everything to reach a steady state. This is normal. + + +$Id$ + +Copyright (C) 2009 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +from __future__ import with_statement + +import lxml.etree, base64, subprocess, sys, os, time, re, getopt, warnings +import rpki.https, rpki.config, rpki.resource_set, rpki.relaxng +import rpki.exceptions, rpki.left_right, rpki.log, rpki.x509, rpki.async +import myrpki, schema + +# Silence warning while loading MySQLdb in Python 2.6, sigh +if hasattr(warnings, "catch_warnings"): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + import MySQLdb +else: + import MySQLdb + +def tag(t): + """ + Wrap an element name in the right XML namespace goop. + """ + return "{http://www.hactrn.net/uris/rpki/myrpki/}" + t + +def findbase64(tree, name, b64type = rpki.x509.X509): + """ + Find and extract a base64-encoded XML element, if present. + """ + x = tree.findtext(tag(name)) + return b64type(Base64 = x) if x else None + +# For simple cases we don't really care what these value are, so long +# as we're consistant about them, so wiring them in is fine. + +bsc_handle = "bsc" +repository_handle = "repository" + +class caller(object): + """ + Handle client-side mechanics for left-right and publication + protocols. + """ + + debug = True + + def __init__(self, proto, client_key, client_cert, server_ta, server_cert, url): + self.proto = proto + self.client_key = client_key + self.client_cert = client_cert + self.server_ta = server_ta + self.server_cert = server_cert + self.url = url + + def __call__(self, cb, eb, pdus): + + def done(cms): + msg, xml = self.proto.cms_msg.unwrap(cms, (self.server_ta, self.server_cert), pretty_print = True) + if self.debug: + print "Reply:", xml + cb(msg) + + msg = self.proto.msg.query(*pdus) + cms, xml = self.proto.cms_msg.wrap(msg, self.client_key, self.client_cert, pretty_print = True) + if self.debug: + print "Query:", xml + + rpki.https.client( + client_key = self.client_key, + client_cert = self.client_cert, + server_ta = self.server_ta, + url = self.url, + msg = cms, + callback = done, + errback = eb) + +os.environ["TZ"] = "UTC" +time.tzset() + +rpki.log.init("myirbe") + +cfg_file = "myrpki.conf" + +bpki_only = False + +opts, argv = getopt.getopt(sys.argv[1:], "bc:h?", ["bpki_only", "config=", "help"]) +for o, a in opts: + if o in ("-b", "--bpki_only"): + bpki_only = True + elif o in ("-c", "--config"): + cfg_file = a + elif o in ("-h", "--help", "-?"): + print __doc__ + sys.exit(0) + +cfg = rpki.config.parser(cfg_file, "myirbe") + +cfg.set_global_flags() + +myrpki.openssl = cfg.get("openssl", "openssl", "myrpki") + +handle = cfg.get("handle", cfg.get("handle", "Amnesiac", "myrpki")) + +want_pubd = cfg.getboolean("want_pubd", False) +want_rootd = cfg.getboolean("want_rootd", False) + +bpki_modified = False + +bpki = myrpki.CA(cfg_file, cfg.get("bpki_directory")) +bpki_modified |= bpki.setup(cfg.get("bpki_ta_dn", "/CN=%s BPKI TA" % handle)) +bpki_modified |= bpki.ee( cfg.get("bpki_rpkid_ee_dn", "/CN=%s rpkid EE" % handle), "rpkid") +bpki_modified |= bpki.ee( cfg.get("bpki_irdbd_ee_dn", "/CN=%s irdbd EE" % handle), "irdbd") +bpki_modified |= bpki.ee( cfg.get("bpki_irbe_ee_dn", "/CN=%s irbe EE" % handle), "irbe") +if want_pubd: + bpki_modified |= bpki.ee( cfg.get("bpki_pubd_ee_dn", "/CN=%s pubd EE" % handle), "pubd") +if want_rootd: + bpki_modified |= bpki.ee( cfg.get("bpki_rootd_ee_dn", "/CN=%s rootd EE" % handle), "rootd") + +if bpki_modified: + print "BPKI (re)initialized. You need to (re)start daemons before continuing." + +if bpki_modified or bpki_only: + sys.exit() + +# Default values for CRL parameters are very low, for testing. + +self_crl_interval = cfg.getint("self_crl_interval", 900) +self_regen_margin = cfg.getint("self_regen_margin", 300) +pubd_base = cfg.get("pubd_base").rstrip("/") + "/" +rpkid_base = cfg.get("rpkid_base").rstrip("/") + "/" + +# Nasty regexp for parsing rpkid's up-down service URLs. + +updown_regexp = re.compile(re.escape(rpkid_base) + "up-down/([-A-Z0-9_]+)/([-A-Z0-9_]+)$", re.I) + +# Wrappers to simplify calling rpkid and pubd. + +call_rpkid = rpki.async.sync_wrapper(caller( + proto = rpki.left_right, + client_key = rpki.x509.RSA( PEM_file = bpki.dir + "/irbe.key"), + client_cert = rpki.x509.X509(PEM_file = bpki.dir + "/irbe.cer"), + server_ta = rpki.x509.X509(PEM_file = bpki.cer), + server_cert = rpki.x509.X509(PEM_file = bpki.dir + "/rpkid.cer"), + url = rpkid_base + "left-right")) + +if want_pubd: + + call_pubd = rpki.async.sync_wrapper(caller( + proto = rpki.publication, + client_key = rpki.x509.RSA( PEM_file = bpki.dir + "/irbe.key"), + client_cert = rpki.x509.X509(PEM_file = bpki.dir + "/irbe.cer"), + server_ta = rpki.x509.X509(PEM_file = bpki.cer), + server_cert = rpki.x509.X509(PEM_file = bpki.dir + "/pubd.cer"), + url = pubd_base + "control")) + + # Make sure that pubd's BPKI CRL is up to date. + + call_pubd((rpki.publication.config_elt.make_pdu( + action = "set", + bpki_crl = rpki.x509.CRL(PEM_file = bpki.crl)),)) + +irdbd_cfg = rpki.config.parser(cfg.get("irdbd_conf", cfg_file), "irdbd") + +db = MySQLdb.connect(user = irdbd_cfg.get("sql-username"), + db = irdbd_cfg.get("sql-database"), + passwd = irdbd_cfg.get("sql-password")) + +cur = db.cursor() + +xmlfiles = [] + +# If [myrpki] section is present in config file, run myrpki.py +# internally, as a convenience, and include its output at the head of +# our list of XML files to process. + +if cfg.has_section("myrpki"): + myrpki.main(("-c", cfg_file)) + my_xmlfile = cfg.get("xml_filename", None, "myrpki") + assert my_xmlfile is not None + xmlfiles.append(my_xmlfile) + +# Add any other XML files specified on the command line + +xmlfiles.extend(argv) + +my_handle = None + +for xmlfile in xmlfiles: + + # Parse XML file and validate it against our scheme + + tree = lxml.etree.parse(xmlfile).getroot() + try: + schema.myrpki.assertValid(tree) + except lxml.etree.DocumentInvalid: + print lxml.etree.tostring(tree, pretty_print = True) + raise + + handle = tree.get("handle") + + if xmlfile == my_xmlfile: + my_handle = handle + + # Update IRDB with parsed resource and roa-request data. + + cur.execute( + """ + DELETE + FROM roa_request_prefix + USING roa_request, roa_request_prefix + WHERE roa_request.roa_request_id = roa_request_prefix.roa_request_id AND roa_request.roa_request_handle = %s + """, (handle,)) + + cur.execute("DELETE FROM roa_request WHERE roa_request.roa_request_handle = %s", (handle,)) + + for x in tree.getiterator(tag("roa_request")): + cur.execute("INSERT roa_request (roa_request_handle, asn) VALUES (%s, %s)", (handle, x.get("asn"))) + roa_request_id = cur.lastrowid + for version, prefix_set in ((4, rpki.resource_set.roa_prefix_set_ipv4(x.get("v4"))), (6, rpki.resource_set.roa_prefix_set_ipv6(x.get("v6")))): + if prefix_set: + cur.executemany("INSERT roa_request_prefix (roa_request_id, prefix, prefixlen, max_prefixlen, version) VALUES (%s, %s, %s, %s, %s)", + ((roa_request_id, p.prefix, p.prefixlen, p.max_prefixlen, version) for p in prefix_set)) + + cur.execute( + """ + DELETE + FROM registrant_asn + USING registrant, registrant_asn + WHERE registrant.registrant_id = registrant_asn.registrant_id AND registrant.registry_handle = %s + """ , (handle,)) + + cur.execute( + """ + DELETE FROM registrant_net USING registrant, registrant_net + WHERE registrant.registrant_id = registrant_net.registrant_id AND registrant.registry_handle = %s + """ , (handle,)) + + cur.execute("DELETE FROM registrant WHERE registrant.registry_handle = %s" , (handle,)) + + for x in tree.getiterator(tag("child")): + child_handle = x.get("handle") + asns = rpki.resource_set.resource_set_as(x.get("asns")) + ipv4 = rpki.resource_set.resource_set_ipv4(x.get("v4")) + ipv6 = rpki.resource_set.resource_set_ipv6(x.get("v6")) + + cur.execute("INSERT registrant (registrant_handle, registry_handle, registrant_name, valid_until) VALUES (%s, %s, %s, %s)", + (child_handle, handle, child_handle, rpki.sundial.datetime.fromXMLtime(x.get("valid_until")).to_sql())) + child_id = cur.lastrowid + if asns: + cur.executemany("INSERT registrant_asn (start_as, end_as, registrant_id) VALUES (%s, %s, %s)", + ((a.min, a.max, child_id) for a in asns)) + if ipv4: + cur.executemany("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 4, %s)", + ((a.min, a.max, child_id) for a in ipv4)) + if ipv6: + cur.executemany("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 6, %s)", + ((a.min, a.max, child_id) for a in ipv6)) + + db.commit() + + # Check for certificates before attempting anything else + + hosted_cacert = findbase64(tree, "bpki_ca_certificate") + if not hosted_cacert: + print "Nothing else I can do without a trust anchor for the entity I'm hosting." + continue + + rpkid_xcert = rpki.x509.X509(PEM_file = bpki.fxcert(handle + ".cacert.cer", + hosted_cacert.get_PEM(), + path_restriction = 1)) + + # See what rpkid and pubd already have on file for this entity. + + if want_pubd: + client_pdus = dict((x.client_handle, x) + for x in call_pubd((rpki.publication.client_elt.make_pdu(action = "list"),)) + if isinstance(x, rpki.publication.client_elt)) + + rpkid_reply = call_rpkid(( + rpki.left_right.self_elt.make_pdu( action = "get", tag = "self", self_handle = handle), + rpki.left_right.bsc_elt.make_pdu( action = "list", tag = "bsc", self_handle = handle), + rpki.left_right.repository_elt.make_pdu(action = "list", tag = "repository", self_handle = handle), + rpki.left_right.parent_elt.make_pdu( action = "list", tag = "parent", self_handle = handle), + rpki.left_right.child_elt.make_pdu( action = "list", tag = "child", self_handle = handle))) + + self_pdu = rpkid_reply[0] + bsc_pdus = dict((x.bsc_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.bsc_elt)) + repository_pdus = dict((x.repository_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.repository_elt)) + parent_pdus = dict((x.parent_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.parent_elt)) + child_pdus = dict((x.child_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.child_elt)) + + pubd_query = [] + rpkid_query = [] + + # There should be exactly one <self/> object per hosted entity, by definition + + if (isinstance(self_pdu, rpki.left_right.report_error_elt) or + self_pdu.crl_interval != self_crl_interval or + self_pdu.regen_margin != self_regen_margin or + self_pdu.bpki_cert != rpkid_xcert): + rpkid_query.append(rpki.left_right.self_elt.make_pdu( + action = "create" if isinstance(self_pdu, rpki.left_right.report_error_elt) else "set", + tag = "self", + self_handle = handle, + bpki_cert = rpkid_xcert, + crl_interval = self_crl_interval, + regen_margin = self_regen_margin)) + + # In general we only need one <bsc/> per <self/>. BSC objects are a + # little unusual in that the PKCS #10 subelement is generated by rpkid + # in response to generate_keypair, so there's more of a separation + # between create and set than with other objects. + + bsc_cert = findbase64(tree, "bpki_bsc_certificate") + bsc_crl = findbase64(tree, "bpki_crl", rpki.x509.CRL) + + bsc_pdu = bsc_pdus.pop(bsc_handle, None) + + if bsc_pdu is None: + rpkid_query.append(rpki.left_right.bsc_elt.make_pdu( + action = "create", + tag = "bsc", + self_handle = handle, + bsc_handle = bsc_handle, + generate_keypair = "yes")) + elif bsc_pdu.signing_cert != bsc_cert or bsc_pdu.signing_cert_crl != bsc_crl: + rpkid_query.append(rpki.left_right.bsc_elt.make_pdu( + action = "set", + tag = "bsc", + self_handle = handle, + bsc_handle = bsc_handle, + signing_cert = bsc_cert, + signing_cert_crl = bsc_crl)) + + rpkid_query.extend(rpki.left_right.bsc_elt.make_pdu( + action = "destroy", self_handle = handle, bsc_handle = b) for b in bsc_pdus) + + bsc_req = None + + if bsc_pdu and bsc_pdu.pkcs10_request: + bsc_req = bsc_pdu.pkcs10_request + + # In general we need one <repository/> per publication daemon with + # whom this <self/> has a relationship. In practice there is rarely + # (never?) a good reason for a single <self/> to use multiple + # publication services, so in normal use we only need one + # <repository/> object. If for some reason you really need more + # than this, you'll have to hack. + + repository_cert = findbase64(tree, "bpki_repository_certificate") + if repository_cert: + + repository_pdu = repository_pdus.pop(repository_handle, None) + repository_uri = pubd_base + "client/" + tree.get("repository_handle") + + if (repository_pdu is None or + repository_pdu.bsc_handle != bsc_handle or + repository_pdu.peer_contact_uri != repository_uri or + repository_pdu.bpki_cert != repository_cert): + rpkid_query.append(rpki.left_right.repository_elt.make_pdu( + action = "create" if repository_pdu is None else "set", + tag = repository_handle, + self_handle = handle, + repository_handle = repository_handle, + bsc_handle = bsc_handle, + peer_contact_uri = repository_uri, + bpki_cert = repository_cert)) + + rpkid_query.extend(rpki.left_right.repository_elt.make_pdu( + action = "destroy", self_handle = handle, repository_handle = r) for r in repository_pdus) + + # <parent/> setup code here used to be ridiculously complex. Most + # of the insanity was due to a misguided attempt to deduce pubd + # setup from other data; now that pubd setup is driven by + # pubclients.csv, parent setup should be relatively straightforward, + # but beware of lingering excessive cleverness in anything dealing + # with parent objects in this script. + + for parent in tree.getiterator(tag("parent")): + + parent_handle = parent.get("handle") + parent_pdu = parent_pdus.pop(parent_handle, None) + parent_uri = parent.get("service_uri") + parent_myhandle = parent.get("myhandle") + parent_sia_base = parent.get("sia_base") + parent_cms_cert = findbase64(parent, "bpki_cms_certificate") + parent_https_cert = findbase64(parent, "bpki_https_certificate") + + if (parent_pdu is None or + parent_pdu.bsc_handle != bsc_handle or + parent_pdu.repository_handle != repository_handle or + parent_pdu.peer_contact_uri != parent_uri or + parent_pdu.sia_base != parent_sia_base or + parent_pdu.sender_name != parent_myhandle or + parent_pdu.recipient_name != parent_handle or + parent_pdu.bpki_cms_cert != parent_cms_cert or + parent_pdu.bpki_https_cert != parent_https_cert): + rpkid_query.append(rpki.left_right.parent_elt.make_pdu( + action = "create" if parent_pdu is None else "set", + tag = parent_handle, + self_handle = handle, + parent_handle = parent_handle, + bsc_handle = bsc_handle, + repository_handle = repository_handle, + peer_contact_uri = parent_uri, + sia_base = parent_sia_base, + sender_name = parent_myhandle, + recipient_name = parent_handle, + bpki_cms_cert = parent_cms_cert, + bpki_https_cert = parent_https_cert)) + + rpkid_query.extend(rpki.left_right.parent_elt.make_pdu( + action = "destroy", self_handle = handle, parent_handle = p) for p in parent_pdus) + + # Children are simpler than parents, because they call us, so no URL + # to construct and figuring out what certificate to use is their + # problem, not ours. + + for child in tree.getiterator(tag("child")): + + child_handle = child.get("handle") + child_pdu = child_pdus.pop(child_handle, None) + child_cert = findbase64(child, "bpki_certificate") + + if (child_pdu is None or + child_pdu.bsc_handle != bsc_handle or + child_pdu.bpki_cert != child_cert): + rpkid_query.append(rpki.left_right.child_elt.make_pdu( + action = "create" if child_pdu is None else "set", + tag = child_handle, + self_handle = handle, + child_handle = child_handle, + bsc_handle = bsc_handle, + bpki_cert = child_cert)) + + rpkid_query.extend(rpki.left_right.child_elt.make_pdu( + action = "destroy", self_handle = handle, child_handle = c) for c in child_pdus) + + # Publication setup, used to be inferred (badly) from parent setup, + # now handled explictly via yet another freaking .csv file. + + if want_pubd: + + for client_handle, client_bpki_cert, client_base_uri in myrpki.csv_open(cfg.get("pubclients_csv", "pubclients.csv")): + + if os.path.exists(client_bpki_cert): + + client_pdu = client_pdus.pop(client_handle, None) + + client_bpki_cert = rpki.x509.X509(PEM_file = bpki.xcert(client_bpki_cert)) + + if (client_pdu is None or + client_pdu.base_uri != client_base_uri or + client_pdu.bpki_cert != client_bpki_cert): + pubd_query.append(rpki.publication.client_elt.make_pdu( + action = "create" if client_pdu is None else "set", + client_handle = client_handle, + bpki_cert = client_bpki_cert, + base_uri = client_base_uri)) + + pubd_query.extend(rpki.publication.client_elt.make_pdu( + action = "destroy", client_handle = p) for p in client_pdus) + + # If we changed anything, ship updates off to daemons + + if rpkid_query: + rpkid_reply = call_rpkid(rpkid_query) + bsc_pdus = dict((x.bsc_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.bsc_elt)) + if bsc_handle in bsc_pdus and bsc_pdus[bsc_handle].pkcs10_request: + bsc_req = bsc_pdus[bsc_handle].pkcs10_request + for r in rpkid_reply: + assert not isinstance(r, rpki.left_right.report_error_elt) + + if pubd_query: + assert want_pubd + pubd_reply = call_pubd(pubd_query) + for r in pubd_reply: + assert not isinstance(r, rpki.publication.report_error_elt) + + # Rewrite XML. + + e = tree.find(tag("bpki_bsc_pkcs10")) + if e is None and bsc_req is not None: + e = lxml.etree.SubElement(tree, "bpki_bsc_pkcs10") + elif bsc_req is None: + tree.remove(e) + + if bsc_req is not None: + assert e is not None + e.text = bsc_req.get_Base64() + + # Something weird going on here with lxml linked against recent + # versions of libxml2. Looks like modifying the tree above somehow + # produces validation errors, but it works fine if we convert it to + # a string and parse it again. I'm not seeing any problems with any + # of the other code that uses lxml to do validation, just this one + # place. Weird. Kludge around it for now. + + tree = lxml.etree.fromstring(lxml.etree.tostring(tree)) + + try: + schema.myrpki.assertValid(tree) + except lxml.etree.DocumentInvalid: + print lxml.etree.tostring(tree, pretty_print = True) + raise + + lxml.etree.ElementTree(tree).write(xmlfile + ".tmp", pretty_print = True) + os.rename(xmlfile + ".tmp", xmlfile) + +db.close() diff --git a/myrpki.rototill/myrpki.py b/myrpki.rototill/myrpki.py new file mode 100644 index 00000000..7937521d --- /dev/null +++ b/myrpki.rototill/myrpki.py @@ -0,0 +1,644 @@ +""" +Read an OpenSSL-style config file and a bunch of .csv files to find +out about parents and children and resources and ROA requests, oh my. +Run OpenSSL command line tool to construct BPKI certificates, +including cross-certification of other entities' BPKI certificates. + +Package up all of the above as a single XML file which user can then +ship off to the IRBE. If an XML file already exists, check it for +data coming back from the IRBE (principally PKCS #10 requests for our +BSC) and update it with current data. + +The general idea here is that this one XML file contains all of the +data that needs to be exchanged as part of ordinary update operations; +each party updates it as necessary, then ships it to the other via +some secure channel: carrier pigeon, USB stick, gpg-protected email, +we don't really care. + +This one program is written a little differently from all the other +Python RPKI programs. This one program is intended to run as a +stand-alone script, without the other programs present. It does +require a reasonably up-to-date version of the OpenSSL command line +tool (the one built as a side effect of building rcynic will do), but +it does -not- require POW or any Python libraries beyond what ships +with Python 2.5. So this script uses xml.etree from the Python +standard libraries instead of lxml.etree, which sacrifices XML schema +validation support in favor of portability, and so forth. + +To make things a little weirder, as a convenience to IRBE operators, +this script can itself be loaded as a Python module and invoked as +part of another program. This requires a few minor contortions, but +avoids duplicating common code. + +$Id$ + +Copyright (C) 2009 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +# Only standard Python libraries for this program, please. + +import subprocess, csv, re, os, getopt, sys, ConfigParser, base64 + +from xml.etree.ElementTree import Element, SubElement, ElementTree + +# Our XML namespace. + +namespace = "http://www.hactrn.net/uris/rpki/myrpki/" + +# Dialect for our use of CSV files, here to make it easy to change if +# your site needs to do something different. See doc for the csv +# module in the Python standard libraries for details if you need to +# customize this. + +csv_dialect = csv.get_dialect("excel-tab") + +# Whether to include incomplete entries when rendering to XML. + +allow_incomplete = False + +# Whether to whine about incomplete entries while rendering to XML. + +whine = False + +class comma_set(set): + """ + Minor customization of set(), to provide a print syntax. + """ + + def __str__(self): + return ",".join(self) + +class roa_request(object): + """ + Representation of a ROA request. + """ + + v4re = re.compile("^([0-9]{1,3}\.){3}[0-9]{1,3}/[0-9]+(-[0-9]+)?$", re.I) + v6re = re.compile("^([0-9a-f]{0,4}:){0,15}[0-9a-f]{0,4}/[0-9]+(-[0-9]+)?$", re.I) + + def __init__(self, asn, group): + self.asn = asn + self.group = group + self.v4 = comma_set() + self.v6 = comma_set() + + def __repr__(self): + s = "<%s asn %s group %s" % (self.__class__.__name__, self.asn, self.group) + if self.v4: + s += " v4 %s" % self.v4 + if self.v6: + s += " v6 %s" % self.v6 + return s + ">" + + def add(self, prefix): + """ + Add one prefix to this ROA request. + """ + if self.v4re.match(prefix): + self.v4.add(prefix) + elif self.v6re.match(prefix): + self.v6.add(prefix) + else: + raise RuntimeError, "Bad prefix syntax: %r" % (prefix,) + + def xml(self, e): + """ + Generate XML element represeting representing this ROA request. + """ + SubElement(e, "roa_request", + asn = self.asn, + v4 = str(self.v4), + v6 = str(self.v6)) + +class roa_requests(dict): + """ + Database of ROA requests. + """ + + def add(self, asn, group, prefix): + """ + Add one <ASN, group, prefix> set to ROA request database. + """ + key = (asn, group) + if key not in self: + self[key] = roa_request(asn, group) + self[key].add(prefix) + + def xml(self, e): + """ + Render ROA requests as XML elements. + """ + for r in self.itervalues(): + r.xml(e) + + @classmethod + def from_csv(cls, roa_csv_file): + """ + Parse ROA requests from CSV file. + """ + self = cls() + # format: p/n-m asn group + for pnm, asn, group in csv_open(roa_csv_file): + self.add(asn = asn, group = group, prefix = pnm) + return self + +class child(object): + """ + Representation of one child entity. + """ + + v4re = re.compile("^(([0-9]{1,3}\.){3}[0-9]{1,3}/[0-9]+)|(([0-9]{1,3}\.){3}[0-9]{1,3}-([0-9]{1,3}\.){3}[0-9]{1,3})$", re.I) + v6re = re.compile("^(([0-9a-f]{0,4}:){0,15}[0-9a-f]{0,4}/[0-9]+)|(([0-9a-f]{0,4}:){0,15}[0-9a-f]{0,4}-([0-9a-f]{0,4}:){0,15}[0-9a-f]{0,4})$", re.I) + + def __init__(self, handle): + self.handle = handle + self.asns = comma_set() + self.v4 = comma_set() + self.v6 = comma_set() + self.validity = None + self.bpki_certificate = None + + def __repr__(self): + s = "<%s %s" % (self.__class__.__name__, self.handle) + if self.asns: + s += " asn %s" % self.asns + if self.v4: + s += " v4 %s" % self.v4 + if self.v6: + s += " v6 %s" % self.v6 + if self.validity: + s += " valid %s" % self.validity + if self.bpki_certificate: + s += " cert %s" % self.bpki_certificate + return s + ">" + + def add(self, prefix = None, asn = None, validity = None, bpki_certificate = None): + """ + Add prefix, autonomous system number, validity date, or BPKI + certificate for this child. + """ + if prefix is not None: + if self.v4re.match(prefix): + self.v4.add(prefix) + elif self.v6re.match(prefix): + self.v6.add(prefix) + else: + raise RuntimeError, "Bad prefix syntax: %r" % (prefix,) + if asn is not None: + self.asns.add(asn) + if validity is not None: + self.validity = validity + if bpki_certificate is not None: + self.bpki_certificate = bpki_certificate + + def xml(self, e): + """ + Render this child as an XML element. + """ + complete = self.bpki_certificate and self.validity + if whine and not complete: + print "Incomplete child entry %s" % self + if complete or allow_incomplete: + e = SubElement(e, "child", + handle = self.handle, + valid_until = self.validity, + asns = str(self.asns), + v4 = str(self.v4), + v6 = str(self.v6)) + if self.bpki_certificate: + PEMElement(e, "bpki_certificate", self.bpki_certificate) + +class children(dict): + """ + Database of children. + """ + + def add(self, handle, prefix = None, asn = None, validity = None, bpki_certificate = None): + """ + Add resources to a child, creating the child object if necessary. + """ + if handle not in self: + self[handle] = child(handle) + self[handle].add(prefix = prefix, asn = asn, validity = validity, bpki_certificate = bpki_certificate) + + def xml(self, e): + """ + Render children database to XML. + """ + for c in self.itervalues(): + c.xml(e) + + @classmethod + def from_csv(cls, children_csv_file, prefix_csv_file, asn_csv_file, xcert): + """ + Parse child resources, certificates, and validity dates from CSV files. + """ + self = cls() + # childname date pemfile + for handle, date, pemfile in csv_open(children_csv_file): + self.add(handle = handle, validity = date, bpki_certificate = xcert(pemfile)) + # childname p/n + for handle, pn in csv_open(prefix_csv_file): + self.add(handle = handle, prefix = pn) + # childname asn + for handle, asn in csv_open(asn_csv_file): + self.add(handle = handle, asn = asn) + return self + +class parent(object): + """ + Representation of one parent entity. + """ + + def __init__(self, handle): + self.handle = handle + self.service_uri = None + self.bpki_cms_certificate = None + self.bpki_https_certificate = None + self.myhandle = None + self.sia_base = None + + def __repr__(self): + s = "<%s %s" % (self.__class__.__name__, self.handle) + if self.myhandle: + s += " myhandle %s" % self.myhandle + if self.service_uri: + s += " uri %s" % self.service_uri + if self.sia_base: + s += " sia %s" % self.sia_base + if self.bpki_cms_certificate: + s += " cms %s" % self.bpki_cms_certificate + if self.bpki_https_certificate: + s += " https %s" % self.bpki_https_certificate + return s + ">" + + def add(self, service_uri = None, + bpki_cms_certificate = None, + bpki_https_certificate = None, + myhandle = None, + sia_base = None): + """ + Add service URI or BPKI certificates to this parent object. + """ + if service_uri is not None: + self.service_uri = service_uri + if bpki_cms_certificate is not None: + self.bpki_cms_certificate = bpki_cms_certificate + if bpki_https_certificate is not None: + self.bpki_https_certificate = bpki_https_certificate + if myhandle is not None: + self.myhandle = myhandle + if sia_base is not None: + self.sia_base = sia_base + + def xml(self, e): + """ + Render this parent object to XML. + """ + complete = self.bpki_cms_certificate and self.bpki_https_certificate and self.myhandle and self.service_uri and self.sia_base + if whine and not complete: + print "Incomplete parent entry %s" % self + if complete or allow_incomplete: + e = SubElement(e, "parent", + handle = self.handle, + myhandle = self.myhandle, + service_uri = self.service_uri, + sia_base = self.sia_base) + if self.bpki_cms_certificate: + PEMElement(e, "bpki_cms_certificate", self.bpki_cms_certificate) + if self.bpki_https_certificate: + PEMElement(e, "bpki_https_certificate", self.bpki_https_certificate) + +class parents(dict): + """ + Database of parent objects. + """ + + def add(self, handle, + service_uri = None, + bpki_cms_certificate = None, + bpki_https_certificate = None, + myhandle = None, + sia_base = None): + """ + Add service URI or certificates to parent object, creating it if necessary. + """ + if handle not in self: + self[handle] = parent(handle) + self[handle].add(service_uri = service_uri, + bpki_cms_certificate = bpki_cms_certificate, + bpki_https_certificate = bpki_https_certificate, + myhandle = myhandle, + sia_base = sia_base) + + def xml(self, e): + for c in self.itervalues(): + c.xml(e) + + @classmethod + def from_csv(cls, parents_csv_file, xcert): + """ + Parse parent data from CSV file. + """ + self = cls() + # parentname service_uri parent_bpki_cms_pemfile parent_bpki_https_pemfile myhandle sia_base + for handle, service_uri, parent_cms_pemfile, parent_https_pemfile, myhandle, sia_base in csv_open(parents_csv_file): + self.add(handle = handle, + service_uri = service_uri, + bpki_cms_certificate = xcert(parent_cms_pemfile), + bpki_https_certificate = xcert(parent_https_pemfile), + myhandle = myhandle, + sia_base = sia_base) + return self + +def csv_open(filename): + """ + Open a CSV file, with settings that make it a tab-delimited file. + You may need to tweak this function for your environment, see the + csv module in the Python standard libraries for details. + """ + return csv.reader(open(filename, "rb"), dialect = csv_dialect) + +def PEMElement(e, tag, filename): + """ + Create an XML element containing Base64 encoded data taken from a + PEM file. + """ + lines = open(filename).readlines() + while lines: + if lines.pop(0).startswith("-----BEGIN "): + break + while lines: + if lines.pop(-1).startswith("-----END "): + break + SubElement(e, tag).text = "".join(line.strip() for line in lines) + +class CA(object): + """ + Representation of one certification authority. + """ + + # Mapping of path restriction values we use to OpenSSL config file + # section names. + + path_restriction = { 0 : "ca_x509_ext_xcert0", + 1 : "ca_x509_ext_xcert1" } + + def __init__(self, cfg, dir): + self.cfg = cfg + self.dir = dir + self.cer = dir + "/ca.cer" + self.key = dir + "/ca.key" + self.req = dir + "/ca.req" + self.crl = dir + "/ca.crl" + self.index = dir + "/index" + self.serial = dir + "/serial" + self.crlnum = dir + "/crl_number" + + self.env = { "PATH" : os.environ["PATH"], + "BPKI_DIRECTORY" : dir, + "RANDFILE" : ".OpenSSL.whines.unless.I.set.this" } + + def run_ca(self, *args): + """ + Run OpenSSL "ca" command with tailored environment variables and common initial + arguments. + """ + cmd = (openssl, "ca", "-batch", "-config", self.cfg) + args + subprocess.check_call(cmd, env = self.env) + + def run_req(self, key_file, req_file): + """ + Run OpenSSL "req" command with tailored environment variables and common arguments. + """ + if not os.path.exists(key_file) or not os.path.exists(req_file): + subprocess.check_call((openssl, "req", "-new", "-sha256", "-newkey", "rsa:2048", + "-config", self.cfg, "-keyout", key_file, "-out", req_file), + env = self.env) + + @staticmethod + def touch_file(filename, content = None): + """ + Create dumb little text files expected by OpenSSL "ca" utility. + """ + if not os.path.exists(filename): + f = open(filename, "w") + if content is not None: + f.write(content) + f.close() + + def setup(self, ca_name): + """ + Set up this CA. ca_name is an X.509 distinguished name in + /tag=val/tag=val format. + """ + + modified = False + + if not os.path.exists(self.dir): + os.makedirs(self.dir) + self.touch_file(self.index) + self.touch_file(self.serial, "01\n") + self.touch_file(self.crlnum, "01\n") + + self.run_req(key_file = self.key, req_file = self.req) + + if not os.path.exists(self.cer): + modified = True + self.run_ca("-selfsign", "-extensions", "ca_x509_ext_ca", "-subj", ca_name, "-in", self.req, "-out", self.cer) + + if not os.path.exists(self.crl): + modified = True + self.run_ca("-gencrl", "-out", self.crl) + + return modified + + def ee(self, ee_name, base_name): + """ + Issue an end-enity certificate. + """ + key_file = "%s/%s.key" % (self.dir, base_name) + req_file = "%s/%s.req" % (self.dir, base_name) + cer_file = "%s/%s.cer" % (self.dir, base_name) + self.run_req(key_file = key_file, req_file = req_file) + if not os.path.exists(cer_file): + self.run_ca("-extensions", "ca_x509_ext_ee", "-subj", ee_name, "-in", req_file, "-out", cer_file) + return True + else: + return False + + def bsc(self, pkcs10): + """ + Issue BSC certificiate, if we have a PKCS #10 request for it. + """ + + if pkcs10 is None: + return None, None + + pkcs10 = base64.b64decode(pkcs10) + + assert pkcs10 + + p = subprocess.Popen((openssl, "dgst", "-md5"), stdin = subprocess.PIPE, stdout = subprocess.PIPE) + hash = p.communicate(pkcs10)[0].strip() + if p.wait() != 0: + raise RuntimeError, "Couldn't hash PKCS#10 request" + + req_file = "%s/bsc.%s.req" % (self.dir, hash) + cer_file = "%s/bsc.%s.cer" % (self.dir, hash) + + if not os.path.exists(cer_file): + + p = subprocess.Popen((openssl, "req", "-inform", "DER", "-out", req_file), stdin = subprocess.PIPE) + p.communicate(pkcs10) + if p.wait() != 0: + raise RuntimeError, "Couldn't store PKCS #10 request" + + self.run_ca("-extensions", "ca_x509_ext_ee", "-in", req_file, "-out", cer_file) + + return req_file, cer_file + + def fxcert(self, filename, cert, path_restriction = 0): + """ + Write PEM certificate to file, then cross-certify. + """ + fn = os.path.join(self.dir, filename) + f = open(fn, "w") + f.write(cert) + f.close() + return self.xcert(fn, path_restriction) + + def xcert(self, cert, path_restriction = 0): + """ + Cross-certify a certificate represented as a PEM file. + """ + + if not cert: + return None + + if not os.path.exists(cert): + #print "Certificate %s doesn't exist, skipping" % cert + return None + + # Extract public key and subject name from PEM file and hash it so + # we can use the result as a tag for cross-certifying this cert. + + p1 = subprocess.Popen((openssl, "x509", "-noout", "-pubkey", "-subject", "-in", cert), stdout = subprocess.PIPE) + p2 = subprocess.Popen((openssl, "dgst", "-md5"), stdin = p1.stdout, stdout = subprocess.PIPE) + + xcert = "%s/xcert.%s.cer" % (self.dir, p2.communicate()[0].strip()) + + if p1.wait() != 0 or p2.wait() != 0: + raise RuntimeError, "Couldn't generate cross-certification tag for %r" % cert + + # Cross-certify the cert we were given, if we haven't already. + # This only works for self-signed certs, due to limitations of the + # OpenSSL command line tool, but that suffices for our purposes. + + if not os.path.exists(xcert): + self.run_ca("-ss_cert", cert, "-out", xcert, "-extensions", self.path_restriction[path_restriction]) + + return xcert + +def extract_resources(): + """ + Extract RFC 3779 resources from a certificate. Not written yet. + + """ + raise NotImplementedError + + +def main(argv = ()): + """ + Main program. Must be callable from other programs as well as being + invoked directly when this module is run as a script. + """ + + cfg_file = "myrpki.conf" + section = "myrpki" + + opts, argv = getopt.getopt(argv, "c:h:?", ["config=", "help"]) + for o, a in opts: + if o in ("-h", "--help", "-?"): + print __doc__ + sys.exit(0) + elif o in ("-c", "--config"): + cfg_file = a + if argv: + raise RuntimeError, "Unexpected arguments %r" % (argv,) + + cfg = ConfigParser.RawConfigParser() + cfg.readfp(open(cfg_file, "r"), cfg_file) + + my_handle = cfg.get(section, "handle") + roa_csv_file = cfg.get(section, "roa_csv") + children_csv_file = cfg.get(section, "children_csv") + parents_csv_file = cfg.get(section, "parents_csv") + prefix_csv_file = cfg.get(section, "prefix_csv") + asn_csv_file = cfg.get(section, "asn_csv") + bpki_dir = cfg.get(section, "bpki_directory") + xml_filename = cfg.get(section, "xml_filename") + repository_bpki_certificate = cfg.get(section, "repository_bpki_certificate") + repository_handle = cfg.get(section, "repository_handle") + + global openssl + openssl = cfg.get(section, "openssl") if cfg.has_option(section, "openssl") else "openssl" + + bpki = CA(cfg_file, bpki_dir) + bpki.setup("/CN=%s TA" % my_handle) + + if os.path.exists(xml_filename): + e = ElementTree(file = xml_filename).getroot() + bsc_req, bsc_cer = bpki.bsc(e.findtext("{%s}%s" % (namespace, "bpki_bsc_pkcs10"))) + else: + bsc_req, bsc_cer = None, None + + e = Element("myrpki", xmlns = namespace, version = "1", handle = my_handle, repository_handle = repository_handle) + + roa_requests.from_csv(roa_csv_file).xml(e) + + children.from_csv( + children_csv_file = children_csv_file, + prefix_csv_file = prefix_csv_file, + asn_csv_file = asn_csv_file, + xcert = bpki.xcert).xml(e) + + parents.from_csv( + parents_csv_file = parents_csv_file, + xcert = bpki.xcert).xml(e) + + PEMElement(e, "bpki_ca_certificate", bpki.cer) + PEMElement(e, "bpki_crl", bpki.crl) + + if os.path.exists(repository_bpki_certificate): + PEMElement(e, "bpki_repository_certificate", bpki.xcert(repository_bpki_certificate)) + + if bsc_cer: + PEMElement(e, "bpki_bsc_certificate", bsc_cer) + + if bsc_req: + PEMElement(e, "bpki_bsc_pkcs10", bsc_req) + + # I still miss SYSCAL(RENMWO) + + ElementTree(e).write(xml_filename + ".tmp") + os.rename(xml_filename + ".tmp", xml_filename) + +# When this file is run as a script, run main() with command line +# arguments. main() can't use sys.argv directly as that might be the +# command line for some other program that loads this module. + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/myrpki.rototill/rcynic.conf b/myrpki.rototill/rcynic.conf new file mode 100644 index 00000000..02a2495b --- /dev/null +++ b/myrpki.rototill/rcynic.conf @@ -0,0 +1,11 @@ +# $Id$ + +[rcynic] +xml-summary = rcynic.xml +jitter = 0 +use-links = yes +use-syslog = no +use-stderr = yes +log-level = log_debug + +trust-anchor = test/RIR/publication/root.cer diff --git a/myrpki.rototill/ripe-to-csv.py b/myrpki.rototill/ripe-to-csv.py new file mode 100644 index 00000000..8166d682 --- /dev/null +++ b/myrpki.rototill/ripe-to-csv.py @@ -0,0 +1,133 @@ +""" +Parse a WHOIS research dump and write out (just) the RPKI-relevant +fields in myrpki-format CSV syntax. + +NB: The input data for this script is publicly available via FTP, but +you'll have to fetch the data from RIPE yourself, and be sure to see +the terms and conditions referenced by the data file header comments. + +$Id$ + +Copyright (C) 2009 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import gzip, csv, myrpki + +class Handle(dict): + + want_tags = () + + debug = True + + def set(self, tag, val): + if tag in self.want_tags: + self[tag] = "".join(val.split(" ")) + + def check(self): + for tag in self.want_tags: + if not tag in self: + return False + if self.debug: + self.log() + return True + + def __repr__(self): + return "<%s %s>" % (self.__class__.__name__, + " ".join("%s:%s" % (tag, self.get(tag, "?")) + for tag in self.want_tags)) + + def log(self): + print repr(self) + + def finish(self, ctx): + self.check() + +class as_block(Handle): + # This one is less useful than I had hoped, no useful links to owners + want_tags = ("as-block", "mnt-by", "org", "mnt-lower") + +class as_set(Handle): + # This is probably useless + want_tags = ("as-set", "mnt-by", "members") + +class aut_num(Handle): + want_tags = ("aut-num", "mnt-by", "as-name") + + def set(self, tag, val): + if tag == "aut-num" and val.startswith("AS"): + val = val[2:] + Handle.set(self, tag, val) + + def finish(self, ctx): + if self.check(): + ctx.asns.writerow((self["mnt-by"], self["aut-num"])) + +class inetnum(Handle): + want_tags = ("inetnum", "mnt-by", "netname") + + def finish(self, ctx): + if self.check(): + ctx.prefixes.writerow((self["mnt-by"], self["inetnum"])) + +class inet6num(Handle): + want_tags = ("inet6num", "mnt-by", "netname") + + def finish(self, ctx): + if self.check(): + ctx.prefixes.writerow((self["mnt-by"], self["inet6num"])) + +class main(object): + + types = dict((x.want_tags[0], x) for x in (as_block, as_set, aut_num, inetnum, inet6num)) + + @staticmethod + def csvout(fn): + return csv.writer(open(fn, "w"), dialect = myrpki.csv_dialect) + + def finish_statement(self, done): + if self.statement: + tag, sep, val = self.statement.partition(":") + assert sep, "Couldn't find separator in %r" % self.statement + tag = tag.strip().lower() + val = val.strip().upper() + if self.cur is None: + self.cur = self.types[tag]() if tag in self.types else False + if self.cur is not False: + self.cur.set(tag, val) + if done and self.cur: + self.cur.finish(self) + self.cur = None + + #filenames = ("ripe.db.gz",) + filenames = ("ripe.db.aut-num.gz", "ripe.db.inet6num.gz", "ripe.db.inetnum.gz") + + def __init__(self): + self.asns = self.csvout("asns.csv") + self.prefixes = self.csvout("prefixes.csv") + for fn in self.filenames: + f = gzip.open(fn) + self.statement = "" + self.cur = None + for line in f: + line = line.expandtabs().partition("#")[0].rstrip("\n") + if line and not line[0].isalpha(): + self.statement += line[1:] if line[0] == "+" else line + else: + self.finish_statement(not line) + self.statement = line + self.finish_statement(True) + f.close() + +main() diff --git a/myrpki.rototill/rpki b/myrpki.rototill/rpki new file mode 120000 index 00000000..168548eb --- /dev/null +++ b/myrpki.rototill/rpki @@ -0,0 +1 @@ +../rpkid/rpki
\ No newline at end of file diff --git a/myrpki.rototill/schema.py b/myrpki.rototill/schema.py new file mode 100644 index 00000000..c371b45b --- /dev/null +++ b/myrpki.rototill/schema.py @@ -0,0 +1,199 @@ +import lxml.etree +myrpki = lxml.etree.RelaxNG(lxml.etree.fromstring('''<?xml version="1.0" encoding="UTF-8"?> +<!-- + $Id: schema.rnc 2839 2009-10-27 18:53:00Z sra $ + + RelaxNG Schema for MyRPKI XML messages + + libxml2 (including xmllint) only groks the XML syntax of RelaxNG, so + run the compact syntax through trang to get XML syntax. +--> +<grammar ns="http://www.hactrn.net/uris/rpki/myrpki/" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes"> + <define name="base64"> + <data type="base64Binary"> + <param name="maxLength">512000</param> + </data> + </define> + <define name="object_handle"> + <data type="string"> + <param name="maxLength">255</param> + <param name="pattern">[\-_A-Za-z0-9]*</param> + </data> + </define> + <define name="pubd_handle"> + <data type="string"> + <param name="maxLength">255</param> + <param name="pattern">[\-_A-Za-z0-9/]*</param> + </data> + </define> + <define name="uri"> + <data type="anyURI"> + <param name="maxLength">4096</param> + </data> + </define> + <define name="asn_list"> + <data type="string"> + <param name="maxLength">512000</param> + <param name="pattern">[\-,0-9]*</param> + </data> + </define> + <define name="ipv4_list"> + <data type="string"> + <param name="maxLength">512000</param> + <param name="pattern">[\-,0-9/.]*</param> + </data> + </define> + <define name="ipv6_list"> + <data type="string"> + <param name="maxLength">512000</param> + <param name="pattern">[\-,0-9/:a-fA-F]*</param> + </data> + </define> + <start> + <element name="myrpki"> + <attribute name="version"> + <data type="positiveInteger"> + <param name="maxInclusive">1</param> + </data> + </attribute> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <attribute name="repository_handle"> + <ref name="pubd_handle"/> + </attribute> + <zeroOrMore> + <ref name="roa_request_elt"/> + </zeroOrMore> + <zeroOrMore> + <ref name="child_elt"/> + </zeroOrMore> + <zeroOrMore> + <ref name="parent_elt"/> + </zeroOrMore> + <optional> + <ref name="bpki_ca_certificate_elt"/> + </optional> + <optional> + <ref name="bpki_crl_elt"/> + </optional> + <optional> + <ref name="bpki_repository_certificate_elt"/> + </optional> + <optional> + <ref name="bpki_bsc_certificate_elt"/> + </optional> + <optional> + <ref name="bpki_bsc_pkcs10_elt"/> + </optional> + </element> + </start> + <define name="roa_request_elt"> + <element name="roa_request"> + <attribute name="asn"> + <data type="positiveInteger"/> + </attribute> + <attribute name="v4"> + <ref name="ipv4_list"/> + </attribute> + <attribute name="v6"> + <ref name="ipv6_list"/> + </attribute> + </element> + </define> + <define name="child_elt"> + <element name="child"> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <attribute name="valid_until"> + <data type="dateTime"> + <param name="pattern">.*Z</param> + </data> + </attribute> + <optional> + <attribute name="asns"> + <ref name="asn_list"/> + </attribute> + </optional> + <optional> + <attribute name="v4"> + <ref name="ipv4_list"/> + </attribute> + </optional> + <optional> + <attribute name="v6"> + <ref name="ipv6_list"/> + </attribute> + </optional> + <optional> + <element name="bpki_certificate"> + <ref name="base64"/> + </element> + </optional> + </element> + </define> + <define name="parent_elt"> + <element name="parent"> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <optional> + <attribute name="service_uri"> + <ref name="uri"/> + </attribute> + </optional> + <optional> + <attribute name="myhandle"> + <ref name="object_handle"/> + </attribute> + </optional> + <optional> + <attribute name="sia_base"> + <ref name="uri"/> + </attribute> + </optional> + <optional> + <element name="bpki_cms_certificate"> + <ref name="base64"/> + </element> + </optional> + <optional> + <element name="bpki_https_certificate"> + <ref name="base64"/> + </element> + </optional> + </element> + </define> + <define name="bpki_ca_certificate_elt"> + <element name="bpki_ca_certificate"> + <ref name="base64"/> + </element> + </define> + <define name="bpki_crl_elt"> + <element name="bpki_crl"> + <ref name="base64"/> + </element> + </define> + <define name="bpki_repository_certificate_elt"> + <element name="bpki_repository_certificate"> + <ref name="base64"/> + </element> + </define> + <define name="bpki_bsc_certificate_elt"> + <element name="bpki_bsc_certificate"> + <ref name="base64"/> + </element> + </define> + <define name="bpki_bsc_pkcs10_elt"> + <element name="bpki_bsc_pkcs10"> + <ref name="base64"/> + </element> + </define> +</grammar> +<!-- + Local Variables: + indent-tabs-mode: nil + End: +--> +''')) diff --git a/myrpki.rototill/schema.rnc b/myrpki.rototill/schema.rnc new file mode 100644 index 00000000..8ec48195 --- /dev/null +++ b/myrpki.rototill/schema.rnc @@ -0,0 +1,64 @@ +# $Id$ +# +# RelaxNG Schema for MyRPKI XML messages +# +# libxml2 (including xmllint) only groks the XML syntax of RelaxNG, so +# run the compact syntax through trang to get XML syntax. + +default namespace = "http://www.hactrn.net/uris/rpki/myrpki/" + +base64 = xsd:base64Binary { maxLength="512000" } +object_handle = xsd:string { maxLength="255" pattern="[\-_A-Za-z0-9]*" } +pubd_handle = xsd:string { maxLength="255" pattern="[\-_A-Za-z0-9/]*" } +uri = xsd:anyURI { maxLength="4096" } +asn_list = xsd:string { maxLength="512000" pattern="[\-,0-9]*" } +ipv4_list = xsd:string { maxLength="512000" pattern="[\-,0-9/.]*" } +ipv6_list = xsd:string { maxLength="512000" pattern="[\-,0-9/:a-fA-F]*" } + +start = element myrpki { + attribute version { xsd:positiveInteger { maxInclusive="1" } }, + attribute handle { object_handle }, + attribute repository_handle { pubd_handle }, + roa_request_elt*, + child_elt*, + parent_elt*, + bpki_ca_certificate_elt?, + bpki_crl_elt?, + bpki_repository_certificate_elt?, + bpki_bsc_certificate_elt?, + bpki_bsc_pkcs10_elt? +} + +roa_request_elt = element roa_request { + attribute asn { xsd:positiveInteger }, + attribute v4 { ipv4_list }, + attribute v6 { ipv6_list } +} + +child_elt = element child { + attribute handle { object_handle }, + attribute valid_until { xsd:dateTime { pattern=".*Z" } }, + attribute asns { asn_list }?, + attribute v4 { ipv4_list }?, + attribute v6 { ipv6_list }?, + element bpki_certificate { base64 }? +} + +parent_elt = element parent { + attribute handle { object_handle }, + attribute service_uri { uri }?, + attribute myhandle { object_handle }?, + attribute sia_base { uri }?, + element bpki_cms_certificate { base64 }?, + element bpki_https_certificate { base64 }? +} + +bpki_ca_certificate_elt = element bpki_ca_certificate { base64 } +bpki_crl_elt = element bpki_crl { base64 } +bpki_repository_certificate_elt = element bpki_repository_certificate { base64 } +bpki_bsc_certificate_elt = element bpki_bsc_certificate { base64 } +bpki_bsc_pkcs10_elt = element bpki_bsc_pkcs10 { base64 } + +# Local Variables: +# indent-tabs-mode: nil +# End: diff --git a/myrpki.rototill/schema.rng b/myrpki.rototill/schema.rng new file mode 100644 index 00000000..6f37e37a --- /dev/null +++ b/myrpki.rototill/schema.rng @@ -0,0 +1,197 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + $Id: schema.rnc 2839 2009-10-27 18:53:00Z sra $ + + RelaxNG Schema for MyRPKI XML messages + + libxml2 (including xmllint) only groks the XML syntax of RelaxNG, so + run the compact syntax through trang to get XML syntax. +--> +<grammar ns="http://www.hactrn.net/uris/rpki/myrpki/" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes"> + <define name="base64"> + <data type="base64Binary"> + <param name="maxLength">512000</param> + </data> + </define> + <define name="object_handle"> + <data type="string"> + <param name="maxLength">255</param> + <param name="pattern">[\-_A-Za-z0-9]*</param> + </data> + </define> + <define name="pubd_handle"> + <data type="string"> + <param name="maxLength">255</param> + <param name="pattern">[\-_A-Za-z0-9/]*</param> + </data> + </define> + <define name="uri"> + <data type="anyURI"> + <param name="maxLength">4096</param> + </data> + </define> + <define name="asn_list"> + <data type="string"> + <param name="maxLength">512000</param> + <param name="pattern">[\-,0-9]*</param> + </data> + </define> + <define name="ipv4_list"> + <data type="string"> + <param name="maxLength">512000</param> + <param name="pattern">[\-,0-9/.]*</param> + </data> + </define> + <define name="ipv6_list"> + <data type="string"> + <param name="maxLength">512000</param> + <param name="pattern">[\-,0-9/:a-fA-F]*</param> + </data> + </define> + <start> + <element name="myrpki"> + <attribute name="version"> + <data type="positiveInteger"> + <param name="maxInclusive">1</param> + </data> + </attribute> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <attribute name="repository_handle"> + <ref name="pubd_handle"/> + </attribute> + <zeroOrMore> + <ref name="roa_request_elt"/> + </zeroOrMore> + <zeroOrMore> + <ref name="child_elt"/> + </zeroOrMore> + <zeroOrMore> + <ref name="parent_elt"/> + </zeroOrMore> + <optional> + <ref name="bpki_ca_certificate_elt"/> + </optional> + <optional> + <ref name="bpki_crl_elt"/> + </optional> + <optional> + <ref name="bpki_repository_certificate_elt"/> + </optional> + <optional> + <ref name="bpki_bsc_certificate_elt"/> + </optional> + <optional> + <ref name="bpki_bsc_pkcs10_elt"/> + </optional> + </element> + </start> + <define name="roa_request_elt"> + <element name="roa_request"> + <attribute name="asn"> + <data type="positiveInteger"/> + </attribute> + <attribute name="v4"> + <ref name="ipv4_list"/> + </attribute> + <attribute name="v6"> + <ref name="ipv6_list"/> + </attribute> + </element> + </define> + <define name="child_elt"> + <element name="child"> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <attribute name="valid_until"> + <data type="dateTime"> + <param name="pattern">.*Z</param> + </data> + </attribute> + <optional> + <attribute name="asns"> + <ref name="asn_list"/> + </attribute> + </optional> + <optional> + <attribute name="v4"> + <ref name="ipv4_list"/> + </attribute> + </optional> + <optional> + <attribute name="v6"> + <ref name="ipv6_list"/> + </attribute> + </optional> + <optional> + <element name="bpki_certificate"> + <ref name="base64"/> + </element> + </optional> + </element> + </define> + <define name="parent_elt"> + <element name="parent"> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <optional> + <attribute name="service_uri"> + <ref name="uri"/> + </attribute> + </optional> + <optional> + <attribute name="myhandle"> + <ref name="object_handle"/> + </attribute> + </optional> + <optional> + <attribute name="sia_base"> + <ref name="uri"/> + </attribute> + </optional> + <optional> + <element name="bpki_cms_certificate"> + <ref name="base64"/> + </element> + </optional> + <optional> + <element name="bpki_https_certificate"> + <ref name="base64"/> + </element> + </optional> + </element> + </define> + <define name="bpki_ca_certificate_elt"> + <element name="bpki_ca_certificate"> + <ref name="base64"/> + </element> + </define> + <define name="bpki_crl_elt"> + <element name="bpki_crl"> + <ref name="base64"/> + </element> + </define> + <define name="bpki_repository_certificate_elt"> + <element name="bpki_repository_certificate"> + <ref name="base64"/> + </element> + </define> + <define name="bpki_bsc_certificate_elt"> + <element name="bpki_bsc_certificate"> + <ref name="base64"/> + </element> + </define> + <define name="bpki_bsc_pkcs10_elt"> + <element name="bpki_bsc_pkcs10"> + <ref name="base64"/> + </element> + </define> +</grammar> +<!-- + Local Variables: + indent-tabs-mode: nil + End: +--> diff --git a/myrpki.rototill/setup-rootd.sh b/myrpki.rototill/setup-rootd.sh new file mode 100644 index 00000000..be7d9368 --- /dev/null +++ b/myrpki.rototill/setup-rootd.sh @@ -0,0 +1,36 @@ +#!/bin/sh - +# +# $Id$ +# +# Copyright (C) 2010 Internet Systems Consortium ("ISC") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +# Setting up rootd requires cross-certifying rpkid's resource-holding +# BPKI trust anchor under the BPKI trust anchor that rootd uses. This +# script handles that, albiet in a very ugly way. +# +# Filenames are wired in, you might need to change these if you've +# done something more complicated. + +export RANDFILE=.OpenSSL.whines.unless.I.set.this +export BPKI_DIRECTORY=`pwd`/bpki.myirbe + +openssl=../openssl/openssl/apps/openssl + +$openssl ca -notext -batch -config myrpki.conf \ + -ss_cert bpki.myrpki/ca.cer \ + -out bpki.myirbe/child.cer \ + -extensions ca_x509_ext_xcert0 + +$openssl x509 -noout -text -in bpki.myirbe/child.cer diff --git a/myrpki.rototill/setup-sql.py b/myrpki.rototill/setup-sql.py new file mode 100644 index 00000000..638404d9 --- /dev/null +++ b/myrpki.rototill/setup-sql.py @@ -0,0 +1,107 @@ +""" +Automated setup of all the pesky SQL stuff we need. Prompts for MySQL +root password, pulls other information from myrpki.conf. + +$Id$ + +Copyright (C) 2009 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +from __future__ import with_statement + +import os, getopt, sys, time, rpki.config, getpass, warnings + +# Silence warning while loading MySQLdb in Python 2.6, sigh +if hasattr(warnings, "catch_warnings"): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + import MySQLdb +else: + import MySQLdb + +import _mysql_exceptions + +warnings.simplefilter("error", _mysql_exceptions.Warning) + +schema_dir = os.path.normpath(os.path.join(sys.path[0], "../rpkid")) + +def read_schema(filename): + """ + Convert an SQL file into a list of SQL statements. + """ + lines = [] + f = open(filename) + for line in f: + line = " ".join(line.split()) + if line and not line.startswith("--"): + lines.append(line) + f.close() + return [statement.strip() for statement in " ".join(lines).rstrip(";").split(";")] + +def sql_setup(name): + """ + Create a new SQL database and construct all its tables. + """ + database = cfg.get("sql-database", section = name) + username = cfg.get("sql-username", section = name) + password = cfg.get("sql-password", section = name) + schema = read_schema(os.path.join(schema_dir, "%s.sql" % name)) + + print "Creating database", database + cur = rootdb.cursor() + try: + cur.execute("DROP DATABASE IF EXISTS %s" % database) + except: + pass + cur.execute("CREATE DATABASE %s" % database) + cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % (database, username), (password,)) + rootdb.commit() + + db = MySQLdb.connect(db = database, user = username, passwd = password) + cur = db.cursor() + for statement in schema: + if statement.upper().startswith("DROP TABLE"): + continue + if verbose: + print "+", statement + cur.execute(statement) + db.commit() + db.close() + +cfg_file = "myrpki.conf" + +verbose = False + +opts, argv = getopt.getopt(sys.argv[1:], "c:hv?", ["config=", "help", "verbose"]) +for o, a in opts: + if o in ("-h", "--help", "-?"): + print __doc__ + sys.exit(0) + if o in ("-v", "--verbose"): + verbose = True + if o in ("-c", "--config"): + cfg_file = a + +cfg = rpki.config.parser(cfg_file, "myirbe") + +rootdb = MySQLdb.connect(db = "mysql", user = "root", passwd = getpass.getpass("Please enter your MySQL root password: ")) + +sql_setup("irdbd") +sql_setup("rpkid") + +if cfg.getboolean("want_pubd", False): + sql_setup("pubd") + +rootdb.close() diff --git a/myrpki.rototill/sql-cleaner.py b/myrpki.rototill/sql-cleaner.py new file mode 100644 index 00000000..8f5f946a --- /dev/null +++ b/myrpki.rototill/sql-cleaner.py @@ -0,0 +1,38 @@ +""" +(Re)Initialize SQL tables used by these programs. + +$Id$ + +Copyright (C) 2009 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import subprocess, ConfigParser + +cfg = ConfigParser.RawConfigParser() +cfg.read("yamltest.conf") + +for name in ("rpkid", "irdbd", "pubd"): + + try: + passwd = cfg.get("yamltest", "%s_db_pass" % name) + except: + passwd = "fnord" + + dbs = [name[:4]] + dbs.extend("%s%d" % (name[:4], i) for i in xrange(12)) + + for db in dbs: + subprocess.check_call(("mysql", "-u", name[:4], "-p" + passwd, db), + stdin = open("../rpkid/%s.sql" % name)) diff --git a/myrpki.rototill/sql-dumper.py b/myrpki.rototill/sql-dumper.py new file mode 100644 index 00000000..849d0eb1 --- /dev/null +++ b/myrpki.rototill/sql-dumper.py @@ -0,0 +1,35 @@ +""" +Dump backup copies of SQL tables used by these programs. + +$Id$ + +Copyright (C) 2009 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import subprocess, ConfigParser + +cfg = ConfigParser.RawConfigParser() +cfg.read("yamltest.conf") + +for name in ("rpkid", "irdbd", "pubd"): + + try: + passwd = cfg.get("yamltest", "%s_db_pass" % name) + except: + passwd = "fnord" + + cmd = ["mysqldump", "-u", name[:4], "-p" + passwd, "--databases", name[:4]] + cmd.extend("%s%d" % (name[:4], i) for i in xrange(12)) + subprocess.check_call(cmd, stdout = open("backup.%s.sql" % name, "w")) diff --git a/myrpki.rototill/start-servers.py b/myrpki.rototill/start-servers.py new file mode 100644 index 00000000..6bd5493e --- /dev/null +++ b/myrpki.rototill/start-servers.py @@ -0,0 +1,73 @@ +""" +Start servers, logging to files, looking at config file to figure out +which servers the user wants started. + +$Id$ + +Copyright (C) 2009 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + +Portions copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + +""" + +import subprocess, os, getopt, sys, time, rpki.config + +rpkid_dir = os.path.normpath(os.path.join(sys.path[0], "../rpkid")) + +os.environ["TZ"] = "UTC" +time.tzset() + +cfg_file = "myrpki.conf" +debug = False + +opts, argv = getopt.getopt(sys.argv[1:], "c:dh?", ["config=", "debug" "help"]) +for o, a in opts: + if o in ("-h", "--help", "-?"): + print __doc__ + sys.exit(0) + elif o in ("-c", "--config"): + cfg_file = a + elif o in ("-d", "--debug"): + debug = True + +names = ["irdbd", "rpkid"] + +cfg = rpki.config.parser(cfg_file, "myirbe") + +if cfg.getboolean("want_pubd", False): + names.append("pubd") + +if cfg.getboolean("want_rootd", False): + names.append("rootd") + +for name in names: + cmd = ("python", os.path.join(rpkid_dir, name + ".py"), "-c", cfg_file) + if debug: + proc = subprocess.Popen(cmd + ("-d",), stdout = open(name + ".log", "a"), stderr = subprocess.STDOUT) + else: + proc = subprocess.Popen(cmd) + print ("Started %r, pid %s" if proc.poll() is None else "Problem starting %r, pid %s") % (name, proc.pid) diff --git a/myrpki.rototill/test-all.sh b/myrpki.rototill/test-all.sh new file mode 100644 index 00000000..1dfc3a52 --- /dev/null +++ b/myrpki.rototill/test-all.sh @@ -0,0 +1,43 @@ +#!/bin/sh - +# $Id$ + +# Copyright (C) 2009 Internet Systems Consortium ("ISC") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +set -x + +export TZ=UTC + +screen -X split +screen -X focus + +for i in ../rpkid/testbed.*.yaml +do + rm -rf *.xml bpki.myrpki bpki.myirbe test + python sql-cleaner.py + screen python yamltest.py $i + date + sleep 180 + for j in . . . . . . . . . . + do + sleep 30 + date + ../rcynic/rcynic + xsltproc --param refresh 0 ../rcynic/rcynic.xsl rcynic.xml | w3m -T text/html -dump + date + done + pstree -ws python | awk '/yamltest/ {system("kill -INT " $2)}' + sleep 30 + make backup +done diff --git a/myrpki.rototill/verify-bpki.sh b/myrpki.rototill/verify-bpki.sh new file mode 100755 index 00000000..9bcf42e6 --- /dev/null +++ b/myrpki.rototill/verify-bpki.sh @@ -0,0 +1,43 @@ +#!/bin/sh - +# $Id$ +# +# Copyright (C) 2009 Internet Systems Consortium ("ISC") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +# Tests of generated BPKI certificates. Kind of cheesy, but does test +# the basic stuff. + +exec 2>&1 + +for bpki in bpki.* +do + crls=$(find $bpki -name '*.crl') + + # Check that CRLs verify properly + for crl in $crls + do + echo -n "$crl: " + openssl crl -CAfile $bpki/ca.cer -noout -in $crl + done + + # Check that issued certificates verify properly + cat $bpki/ca.cer $crls | openssl verify -crl_check -CAfile /dev/stdin $(find $bpki -name '*.cer' ! -name 'ca.cer' ! -name '*.cacert.cer') + +done + +# Check that cross-certified BSC certificates verify properly +if test -d bpki.myirbe +then + cat bpki.myirbe/xcert.*.cer | openssl verify -verbose -CAfile bpki.myirbe/ca.cer -untrusted /dev/stdin bpki.myrpki/bsc.*.cer +fi diff --git a/myrpki.rototill/wsgi-example.py b/myrpki.rototill/wsgi-example.py new file mode 100644 index 00000000..5ae8ad13 --- /dev/null +++ b/myrpki.rototill/wsgi-example.py @@ -0,0 +1,27 @@ +# $Id$ + +# Every WSGI application must have an application object - a callable +# object that accepts two arguments. For that purpose, we're going to +# use a function (note that you're not limited to a function, you can +# use a class for example). The first argument passed to the function +# is a dictionary containing CGI-style envrironment variables and the +# second variable is the callable object (see PEP333) + +# See http://pythonpaste.org/do-it-yourself-framework.html for a +# somewhat more complete introduction, although it's a lead-in to the +# Paste package which we might not want to use. + +def hello_world_app(environ, start_response): + status = '200 OK' # HTTP Status + headers = [('Content-type', 'text/plain')] # HTTP Headers + start_response(status, headers) + + # The returned object is going to be printed + return ["Hello World"] + +# Run server with this app on port 8000 if invoked as a script + +if __name__ == "__main__": + from wsgiref.simple_server import make_server + print "Serving on port 8000..." + make_server('', 8000, hello_world_app).serve_forever() diff --git a/myrpki.rototill/xml-parse-test.py b/myrpki.rototill/xml-parse-test.py new file mode 100644 index 00000000..d5f8e007 --- /dev/null +++ b/myrpki.rototill/xml-parse-test.py @@ -0,0 +1,100 @@ +""" +Test parser and display tool for myrpki.xml files. + +$Id$ + +Copyright (C) 2009 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import lxml.etree, rpki.resource_set, base64, subprocess +import schema + +tree = lxml.etree.parse("myrpki.xml").getroot() + +if False: + print lxml.etree.tostring(tree, pretty_print = True, encoding = "us-ascii", xml_declaration = True) + +schema.myrpki.assertValid(tree) + +def showitems(x): + if False: + for k, v in x.items(): + if v: + print " ", k, v + +def tag(t): + return "{http://www.hactrn.net/uris/rpki/myrpki/}" + t + +print "My handle:", tree.get("handle") + +print "Children:" +for x in tree.getiterator(tag("child")): + print " ", x + print " Handle:", x.get("handle") + print " ASNS: ", rpki.resource_set.resource_set_as(x.get("asns")) + print " IPv4: ", rpki.resource_set.resource_set_ipv4(x.get("v4")) + print " Valid: ", x.get("valid_until") + showitems(x) +print + +print "ROA requests:" +for x in tree.getiterator(tag("roa_request")): + print " ", x + print " ASN: ", x.get("asn") + print " IPv4:", rpki.resource_set.roa_prefix_set_ipv4(x.get("v4")) + print " IPv6:", rpki.resource_set.roa_prefix_set_ipv6(x.get("v6")) + showitems(x) +print + +def showpem(label, b64, kind): + cmd = ("openssl", kind, "-noout", "-text", "-inform", "DER") + if kind == "x509": + cmd += ("-certopt", "no_pubkey,no_sigdump") + p = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE) + text = p.communicate(input = base64.b64decode(b64))[0] + if p.returncode != 0: + raise subprocess.CalledProcessError(returncode = p.returncode, cmd = cmd) + print label, text + +for x in tree.getiterator(tag("child")): + cert = x.findtext(tag("bpki_certificate")) + if cert: + showpem("Child", cert, "x509") + +for x in tree.getiterator(tag("parent")): + print "Parent URI:", x.get("service_uri") + cert = x.findtext(tag("bpki_certificate")) + if cert: + showpem("Parent", cert, "x509") + +ca = tree.findtext(tag("bpki_ca_certificate")) +if ca: + showpem("CA", ca, "x509") + +bsc = tree.findtext(tag("bpki_bsc_certificate")) +if bsc: + showpem("BSC EE", bsc, "x509") + +repo = tree.findtext(tag("bpki_repository_certificate")) +if repo: + showpem("Repository", repo, "x509") + +req = tree.findtext(tag("bpki_bsc_pkcs10")) +if req: + showpem("BSC EE", req, "req") + +crl = tree.findtext(tag("bpki_crl")) +if crl: + showpem("CA", crl, "crl") diff --git a/myrpki.rototill/yamltest.py b/myrpki.rototill/yamltest.py new file mode 100644 index 00000000..6c4f83da --- /dev/null +++ b/myrpki.rototill/yamltest.py @@ -0,0 +1,700 @@ +""" +Test framework, using the same YAML test description format as +testbed.py, but using the myrpki.py and myirbe.py tools to do all the +back-end work. Reads YAML file, generates .csv and .conf files, runs +daemons and waits for one of them to exit. + +Much of the YAML handling code lifted from testbed.py. + +Still to do: + +- Implement testebd.py-style delta actions, that is, modify the + allocation database under control of the YAML file, dump out new + .csv files, and run myrpki.py and myirbe.py again to feed resulting + changes into running daemons. + +$Id$ + +Copyright (C) 2009 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + +Portions copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + +""" + +import subprocess, csv, re, os, getopt, sys, base64, yaml, signal, errno, time +import rpki.resource_set, rpki.sundial, rpki.config, rpki.log, myrpki + +# Nasty regular expressions for parsing config files. Sadly, while +# the Python ConfigParser supports writing config files, it does so in +# such a limited way that it's easier just to hack this ourselves. + +section_regexp = re.compile("\s*\[\s*(.+?)\s*\]\s*$") +variable_regexp = re.compile("\s*([-a-zA-Z0-9_]+)\s*=\s*(.+?)\s*$") + +def cleanpath(*names): + """ + Construct normalized pathnames. + """ + return os.path.normpath(os.path.join(*names)) + +# Pathnames for various things we need + +this_dir = os.getcwd() +test_dir = cleanpath(this_dir, "test") +rpkid_dir = cleanpath(this_dir, "../rpkid") + +prog_myirbe = cleanpath(this_dir, "myirbe.py") +prog_myrpki = cleanpath(this_dir, "myrpki.py") +prog_rpkid = cleanpath(rpkid_dir, "rpkid.py") +prog_irdbd = cleanpath(rpkid_dir, "irdbd.py") +prog_pubd = cleanpath(rpkid_dir, "pubd.py") +prog_rootd = cleanpath(rpkid_dir, "rootd.py") + +prog_openssl = cleanpath(this_dir, "../openssl/openssl/apps/openssl") + +class roa_request(object): + """ + Representation of a ROA request. + """ + + def __init__(self, asn, ipv4, ipv6): + self.asn = asn + self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None + self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None + + def __eq__(self, other): + return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 + + def __hash__(self): + v4 = tuple(self.v4) if self.v4 is not None else None + v6 = tuple(self.v6) if self.v6 is not None else None + return self.asn.__hash__() + v4.__hash__() + v6.__hash__() + + def __str__(self): + if self.v4 and self.v6: + return "%s: %s,%s" % (self.asn, self.v4, self.v6) + else: + return "%s: %s" % (self.asn, self.v4 or self.v6) + + @classmethod + def parse(cls, yaml): + """ + Parse a ROA request from YAML format. + """ + return cls(yaml.get("asn"), yaml.get("ipv4"), yaml.get("ipv6")) + +class allocation_db(list): + """ + Our allocation database. + """ + + def __init__(self, yaml): + list.__init__(self) + self.root = allocation(yaml, self) + assert self.root.is_root() + if self.root.crl_interval is None: + self.root.crl_interval = 24 * 60 * 60 + if self.root.regen_margin is None: + self.root.regen_margin = 24 * 60 * 60 + for a in self: + if a.sia_base is None: + if a.runs_pubd(): + base = "rsync://localhost:%d/" % a.rsync_port + else: + base = a.parent.sia_base + a.sia_base = base + a.name + "/" + if a.base.valid_until is None: + a.base.valid_until = a.parent.base.valid_until + if a.crl_interval is None: + a.crl_interval = a.parent.crl_interval + if a.regen_margin is None: + a.regen_margin = a.parent.regen_margin + i = 0 + for j in xrange(3): + i = a.sia_base.index("/", i) + 1 + a.client_handle = a.sia_base[i:].rstrip("/") + self.root.closure() + self.map = dict((a.name, a) for a in self) + for a in self: + if a.is_hosted(): + a.hosted_by = self.map[a.hosted_by] + a.hosted_by.hosts.append(a) + assert not a.is_root() and not a.hosted_by.is_hosted() + + def dump(self): + """ + Show contents of allocation database. + """ + for a in self: + a.dump() + + def make_rootd_openssl(self): + """ + Factory for a function to run the OpenSSL comand line tool on the + root node of our allocation database. Could easily be generalized + if there were a need, but as it happens we only ever need to do + this for the root node. + """ + env = { "PATH" : os.environ["PATH"], + "BPKI_DIRECTORY" : self.root.path("bpki.myirbe"), + "OPENSSL_CONF" : "/dev/null", + "RANDFILE" : ".OpenSSL.whines.unless.I.set.this" } + cwd = self.root.path() + return lambda *args: subprocess.check_call((prog_openssl,) + args, cwd = cwd, env = env) + +class allocation(object): + """ + One entity in our allocation database. Every entity in the database + is assumed to hold resources, so needs at least myrpki services. + Entities that don't have the hosted_by property run their own copies + of rpkid, irdbd, and pubd, so they also need myirbe services. + """ + + parent = None + crl_interval = None + regen_margin = None + + base_port = 4400 + + @classmethod + def allocate_port(cls): + """ + Allocate a TCP port. + """ + cls.base_port += 1 + return cls.base_port + + base_engine = -1 + + @classmethod + def allocate_engine(cls): + """ + Allocate an engine number, mostly used to construct MySQL database + names. + """ + cls.base_engine += 1 + return cls.base_engine + + def __init__(self, yaml, db, parent = None): + db.append(self) + self.name = yaml["name"] + self.parent = parent + self.kids = [allocation(k, db, self) for k in yaml.get("kids", ())] + valid_until = None + if "valid_until" in yaml: + valid_until = rpki.sundial.datetime.fromdatetime(yaml.get("valid_until")) + if valid_until is None and "valid_for" in yaml: + valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(yaml["valid_for"]) + self.base = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as(yaml.get("asn")), + v4 = rpki.resource_set.resource_set_ipv4(yaml.get("ipv4")), + v6 = rpki.resource_set.resource_set_ipv6(yaml.get("ipv6")), + valid_until = valid_until) + self.sia_base = yaml.get("sia_base") + if "crl_interval" in yaml: + self.crl_interval = rpki.sundial.timedelta.parse(yaml["crl_interval"]).convert_to_seconds() + if "regen_margin" in yaml: + self.regen_margin = rpki.sundial.timedelta.parse(yaml["regen_margin"]).convert_to_seconds() + self.roa_requests = [roa_request.parse(y) for y in yaml.get("roa_request", yaml.get("route_origin", ()))] + for r in self.roa_requests: + if r.v4: + self.base.v4 = self.base.v4.union(r.v4.to_resource_set()) + if r.v6: + self.base.v6 = self.base.v6.union(r.v6.to_resource_set()) + self.hosted_by = yaml.get("hosted_by") + self.hosts = [] + if not self.is_hosted(): + self.engine = self.allocate_engine() + self.rpkid_port = self.allocate_port() + self.irdbd_port = self.allocate_port() + if self.runs_pubd(): + self.pubd_port = self.allocate_port() + self.rsync_port = self.allocate_port() + if self.is_root(): + self.rootd_port = self.allocate_port() + + def closure(self): + """ + Compute resource closure of this node and its children, to avoid a + lot of tedious (and error-prone) duplication in the YAML file. + """ + resources = self.base + for kid in self.kids: + resources = resources.union(kid.closure()) + self.resources = resources + return resources + + def dump(self): + """ + Show content of this allocation node. + """ + print str(self) + + def __str__(self): + s = self.name + ":\n" + if self.resources.asn: s += " ASNs: %s\n" % self.resources.asn + if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 + if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 + if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) + if self.parent: s += " Up: %s\n" % self.parent.name + if self.sia_base: s += " SIA: %s\n" % self.sia_base + if self.is_hosted(): s += " Host: %s\n" % self.hosted_by.name + if self.hosts: s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts) + for r in self.roa_requests: s += " ROA: %s\n" % r + if not self.is_hosted(): s += " IPort: %s\n" % self.irdbd_port + if self.runs_pubd(): s += " PPort: %s\n" % self.pubd_port + if not self.is_hosted(): s += " RPort: %s\n" % self.rpkid_port + if self.runs_pubd(): s += " SPort: %s\n" % self.rsync_port + if self.is_root(): s += " TPort: %s\n" % self.rootd_port + return s + " Until: %s\n" % self.resources.valid_until + + def is_root(self): + """ + Is this the root node? + """ + return self.parent is None + + def is_hosted(self): + """ + Is this entity hosted? + """ + return self.hosted_by is not None + + def runs_pubd(self): + """ + Does this entity run a pubd? + """ + return self.is_root() or not (self.is_hosted() or only_one_pubd) + + def path(self, *names): + """ + Construct pathnames in this entity's test directory. + """ + return cleanpath(test_dir, self.name, *names) + + def csvout(self, fn): + """ + Open and log a CSV output file. We use delimiter and dialect + settings imported from the myrpki module, so that we automatically + write CSV files in the right format. + """ + path = self.path(fn) + print "Writing", path + return csv.writer(open(path, "w"), dialect = myrpki.csv_dialect) + + def up_down_url(self): + """ + Construct service URL for this node's parent. + """ + parent_port = self.parent.hosted_by.rpkid_port if self.parent.is_hosted() else self.parent.rpkid_port + return "https://localhost:%d/up-down/%s/%s" % (parent_port, self.parent.name, self.name) + + def dump_asns(self, fn): + """ + Write Autonomous System Numbers CSV file. + """ + f = self.csvout(fn) + for k in self.kids: + f.writerows((k.name, a) for a in k.resources.asn) + + def dump_children(self, fn): + """ + Write children CSV file. + """ + self.csvout(fn).writerows((k.name, k.resources.valid_until, k.path("bpki.myrpki/ca.cer")) + for k in self.kids) + + def dump_parents(self, fn): + """ + Write parents CSV file. + """ + if self.is_root(): + self.csvout(fn).writerow(("rootd", + "https://localhost:%d/" % self.rootd_port, + self.path("bpki.myirbe/ca.cer"), + self.path("bpki.myirbe/ca.cer"), + self.name, + self.sia_base)) + else: + parent_host = self.parent.hosted_by if self.parent.is_hosted() else self.parent + self.csvout(fn).writerow((self.parent.name, + self.up_down_url(), + self.parent.path("bpki.myrpki/ca.cer"), + parent_host.path("bpki.myirbe/ca.cer"), + self.name, + self.sia_base)) + + def dump_prefixes(self, fn): + """ + Write prefixes CSV file. + """ + f = self.csvout(fn) + for k in self.kids: + f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) + + def dump_roas(self, fn): + """ + Write ROA CSV file. + """ + group = self.name if self.is_root() else self.parent.name + f = self.csvout(fn) + for r in self.roa_requests: + f.writerows((p, r.asn, group) + for p in (r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ())) + + def dump_clients(self, fn, db): + """ + Write pubclients CSV file. + """ + if self.runs_pubd(): + f = self.csvout(fn) + f.writerows((s.client_handle, s.path("bpki.myrpki/ca.cer"), s.sia_base) + for s in (db if only_one_pubd else [self] + self.kids)) + + def dump_conf(self, fn): + """ + Write configuration file for OpenSSL and RPKI tools. + """ + + host = self.hosted_by if self.is_hosted() else self + + r = { ("myrpki", "handle"): self.name } + + if not self.is_hosted(): + r["irdbd", "https-url"] = "https://localhost:%d/" % self.irdbd_port + r["irdbd", "sql-database"] = "irdb%d" % self.engine + r["myirbe", "irdbd_conf"] = "myrpki.conf" + r["myirbe", "rpkid_base"] = "https://localhost:%d/" % self.rpkid_port + r["rpkid", "irdb-url"] = "https://localhost:%d/" % self.irdbd_port + r["rpkid", "server-port"] = "%d" % self.rpkid_port + r["rpkid", "sql-database"] = "rpki%d" % self.engine + r["myirbe", "want_pubd"] = "true" if self.runs_pubd() else "false" + r["myirbe", "want_rootd"] = "true" if self.is_root() else "false" + r["irbe_cli", "rpkid-url"] = "https://localhost:%d/left-right" % self.rpkid_port + + if self.is_root(): + root_path = "localhost:%d/%s" % (self.rsync_port, self.name) + r["rootd", "rpki-root-dir"] = "publication/" + r["rootd", "rpki-base-uri"] = "rsync://%s/" % root_path + r["rootd", "rpki-root-cert"] = "publication/root.cer" + r["rootd", "rpki-root-cert-uri"] = "rsync://%s/root.cer" % root_path + r["rootd", "rpki-subject-cert"] = "%s.cer" % self.name + r["rootd", "rpki-root-manifest"] = "root.mnf" + r["rootd", "root_cert_sia"] = r["rootd", "rpki-base-uri"] + r["rootd", "root_cert_manifest"] = r["rootd", "rpki-base-uri"] + r["rootd", "rpki-root-manifest"] + + if self.runs_pubd(): + r["pubd", "server-port"] = "%d" % self.pubd_port + r["pubd", "sql-database"] = "pubd%d" % self.engine + r["irbe_cli", "pubd-url"] = "https://localhost:%d/control/" % self.pubd_port + + s = self + while not s.runs_pubd(): + s = s.parent + r["myirbe", "pubd_base"] = "https://localhost:%d/" % s.pubd_port + r["myirbe", "rsync_base"] = "rsync://localhost:%d/" % s.rsync_port + r["myrpki", "repository_bpki_certificate"] = s.path("bpki.myirbe/ca.cer") + r["myrpki", "repository_handle"] = self.client_handle + + if self.is_root(): + r["rootd", "server-port"] = "%d" % self.rootd_port + + if rpkid_password: + r["rpkid", "sql-password"] = rpkid_password + + if irdbd_password: + r["irdbd", "sql-password"] = irdbd_password + + if pubd_password: + r["pubd", "sql-password"] = pubd_password + + f = open(self.path(fn), "w") + f.write("# Automatically generated, do not edit\n") + print "Writing", f.name + + section = None + for line in open("examples/myrpki.conf"): + if not line.strip() or line.lstrip().startswith("#"): + continue + m = section_regexp.match(line) + if m: + section = m.group(1) + if (section is None or + (self.is_hosted() and section in ("myirbe", "rpkid", "irdbd")) or + (not self.runs_pubd() and section == "pubd") or + (not self.is_root() and section in ("rootd", "rootd_x509_extensions"))): + continue + m = variable_regexp.match(line) if m is None else None + variable = m.group(1) if m else None + if (section, variable) in r: + line = variable + " = " + r[section, variable] + "\n" + f.write(line) + + f.close() + + def dump_rsyncd(self, fn): + """ + Write rsyncd configuration file. + """ + + if self.runs_pubd(): + f = open(self.path(fn), "w") + print "Writing", f.name + f.writelines(s + "\n" for s in + ("# Automatically generated, do not edit", + "port = %d" % self.rsync_port, + "address = localhost", + "[%s]" % self.name, + "log file = rsyncd.log", + "read only = yes", + "use chroot = no", + "path = %s" % self.path("publication"), + "comment = RPKI test")) + f.close() + + def run_myirbe(self): + """ + Run myirbe.py if this entity is not hosted by another engine. + """ + if not self.is_hosted(): + print "Running myirbe.py for", self.name + cmd = ["python", prog_myirbe] + cmd.extend(h.path("myrpki.xml") for h in self.hosts) + subprocess.check_call(cmd, cwd = self.path()) + + def run_myrpki(self): + """ + Run myrpki.py for this entity. + """ + print "Running myrpki.py for", self.name + subprocess.check_call(("python", prog_myrpki), cwd = self.path()) + + def run_python_daemon(self, prog): + """ + Start a Python daemon and return a subprocess.Popen object + representing the running daemon. + """ + basename = os.path.basename(prog) + p = subprocess.Popen(("python", prog, "-d", "-c", self.path("myrpki.conf")), + cwd = self.path(), + stdout = open(self.path(os.path.splitext(basename)[0] + ".log"), "w"), + stderr = subprocess.STDOUT) + print "Running %s for %s: pid %d process %r" % (basename, self.name, p.pid, p) + return p + + def run_rpkid(self): + """ + Run rpkid. + """ + return self.run_python_daemon(prog_rpkid) + + def run_irdbd(self): + """ + Run irdbd. + """ + return self.run_python_daemon(prog_irdbd) + + def run_pubd(self): + """ + Run pubd. + """ + return self.run_python_daemon(prog_pubd) + + def run_rootd(self): + """ + Run rootd. + """ + return self.run_python_daemon(prog_rootd) + + def run_rsyncd(self): + """ + Run rsyncd. + """ + p = subprocess.Popen(("rsync", "--daemon", "--no-detach", "--config", "rsyncd.conf"), + cwd = self.path()) + print "Running rsyncd for %s: pid %d process %r" % (self.name, p.pid, p) + return p + +os.environ["TZ"] = "UTC" +time.tzset() + +cfg_file = "yamltest.conf" + +opts, argv = getopt.getopt(sys.argv[1:], "c:h?", ["config=", "help"]) +for o, a in opts: + if o in ("-h", "--help", "-?"): + print __doc__ + sys.exit(0) + if o in ("-c", "--config"): + cfg_file = a + +# We can't usefully process more than one YAMl file at a time, so +# whine if there's more than one argument left. + +if len(argv) > 1: + raise RuntimeError, "Unexpected arguments %r" % argv + +rpki.log.use_syslog = False +rpki.log.init("yamltest") + +yaml_file = argv[0] if argv else "../rpkid/testbed.1.yaml" + +# Allow optional config file for this tool to override default +# passwords: this is mostly so that I can show a complete working +# example without publishing my own server's passwords. + +try: + cfg = rpki.config.parser(cfg_file, "yamltest") + rpkid_password = cfg.get("rpkid_db_pass") + irdbd_password = cfg.get("irdbd_db_pass") + pubd_password = cfg.get("pubd_db_pass") + only_one_pubd = cfg.getboolean("only_one_pubd", True) + prog_openssl = cfg.get("openssl", prog_openssl) +except: + rpkid_password = None + irdbd_password = None + pubd_password = None + only_one_pubd = True + +# Start clean + +for root, dirs, files in os.walk(test_dir, topdown = False): + for file in files: + os.unlink(os.path.join(root, file)) + for dir in dirs: + os.rmdir(os.path.join(root, dir)) + +# Read first YAML doc in file and process as compact description of +# test layout and resource allocations. Ignore subsequent YAML docs, +# they're for testbed.py, not this script. + +db = allocation_db(yaml.safe_load_all(open(yaml_file)).next()) + +# Show what we loaded + +db.dump() + +# Set up each entity in our test + +for d in db: + os.makedirs(d.path()) + d.dump_asns("asns.csv") + d.dump_children("children.csv") + d.dump_parents("parents.csv") + d.dump_prefixes("prefixes.csv") + d.dump_roas("roas.csv") + d.dump_conf("myrpki.conf") + d.dump_clients("pubclients.csv", db) + d.dump_rsyncd("rsyncd.conf") + +# Do initial myirbe.py run for each hosting entity to set up BPKI + +for d in db: + d.run_myirbe() + +# Run myrpki.py several times for each entity. First pass misses +# stuff that isn't generated until later in first pass. Second pass +# should pick up everything and reach a stable state. If anything +# changes during third pass, that's a bug. + +for i in xrange(3): + for d in db: + d.run_myrpki() + +# Set up a few things for rootd + +rootd_openssl = db.make_rootd_openssl() + +print "Creating rootd BPKI cross-certificate for its child" +rootd_openssl("ca", "-notext", "-batch", + "-config", "myrpki.conf", + "-ss_cert", "bpki.myrpki/ca.cer", + "-out", "bpki.myirbe/child.cer", + "-extensions", "ca_x509_ext_xcert0") + +os.makedirs(db.root.path("publication")) + +print "Creating rootd RPKI root certificate" +rootd_openssl("x509", "-req", "-sha256", "-outform", "DER", + "-signkey", "bpki.myirbe/ca.key", + "-in", "bpki.myirbe/ca.req", + "-out", "publication/root.cer", + "-extfile", "myrpki.conf", + "-extensions", "rootd_x509_extensions") + +# At this point we need to start a whole lotta daemons. + +progs = [] + +def all_daemons_running(): + for p in progs: + if p.poll() is not None: + return False + return True + +try: + print "Running daemons" + progs.append(db.root.run_rootd()) + progs.extend(d.run_irdbd() for d in db if not d.is_hosted()) + progs.extend(d.run_pubd() for d in db if d.runs_pubd()) + progs.extend(d.run_rsyncd() for d in db if d.runs_pubd()) + progs.extend(d.run_rpkid() for d in db if not d.is_hosted()) + + print "Giving daemons time to start up" + time.sleep(20) + + assert all_daemons_running() + + # Run myirbe again for each host, to set up IRDB and RPKI objects. + # Need to run a second time to push BSC certs out to rpkid. Nothing + # should happen on the third pass. Oops, when hosting we need to + # run myrpki between myirbe passes, since only the hosted entity can + # issue the BSC, etc. + + for i in xrange(3): + for d in db: + d.run_myrpki() + for d in db: + d.run_myirbe() + + print "Done initializing daemons" + + # Wait until something terminates. + + signal.signal(signal.SIGCHLD, lambda *dont_care: None) + if all_daemons_running(): + signal.pause() + +finally: + + # Shut everything down. + + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + for p in progs: + if p.poll() is None: + os.kill(p.pid, signal.SIGTERM) + print "Program pid %d %r returned %d" % (p.pid, p, p.wait()) |