././@PaxHeader 0000000 0000000 0000000 00000000033 00000000000 010211 x ustar 00 27 mtime=1706190263.134136
blaeu-1.1.10/ 0000755 0001750 0001750 00000000000 14554462667 014133 5 ustar 00bortzmeyer bortzmeyer ././@PaxHeader 0000000 0000000 0000000 00000000026 00000000000 010213 x ustar 00 22 mtime=1706173835.0
blaeu-1.1.10/Blaeu.py 0000644 0001750 0001750 00000063105 14554422613 015525 0 ustar 00bortzmeyer bortzmeyer #!/usr/bin/env python3
# -*- coding: utf-8 -*-
""" A module to perform measurements on the RIPE Atlas
probes using the UDM (User Defined
Measurements) creation API.
Authorization key is expected in $HOME/.atlas/auth
in the environment variable ATLASAUTH or have to be
provided in the constructor's arguments.
Stéphane Bortzmeyer
"""
# WARNING: if you modify it here, also change setup.py https://packaging.python.org/guides/single-sourcing-package-version/#single-sourcing-the-version
VERSION = '1.1.10'
import os
import json
import time
import urllib.request, urllib.error, urllib.parse
import random
import copy
import sys
import getopt
import string
authfile = "%s/.atlas/auth" % os.environ['HOME']
base_url = "https://atlas.ripe.net/api/v2/measurements"
# The following parameters are currently not settable. Anyway, be
# careful when changing these, you may get inconsistent results if you
# do not wait long enough. Other warning: the time to wait depend on
# the number of the probes.
# All in seconds:
fields_delay_base = 6
fields_delay_factor = 0.2
results_delay_base = 3
results_delay_factor = 0.15
maximum_time_for_results_base = 30
maximum_time_for_results_factor = 5
# The basic problem is that there is no easy way in Atlas to know when
# it is over, either for retrieving the list of the probes, or for
# retrieving the results themselves. The only solution is to wait
# "long enough". The time to wait is not documented so the values
# above have been found mostly with trial-and-error.
class AuthFileNotFound(Exception):
pass
class AuthFileEmpty(Exception):
pass
class RequestSubmissionError(Exception):
pass
class FieldsQueryError(Exception):
pass
class MeasurementNotFound(Exception):
pass
class MeasurementAccessError(Exception):
pass
class ResultError(Exception):
pass
class IncompatibleArguments(Exception):
pass
class InternalError(Exception):
pass
# Resut JSON file does not have the expected fields/members
class WrongAssumption(Exception):
pass
class Config:
def __init__(self):
# Default values
self.old_measurement = None
self.measurement_id = None
self.probes = None
self.country = None # World-wide
self.asn = None # All
self.area = None # World-wide
self.prefix = None
self.verbose = False
self.requested = 5 # Probes
self.default_requested = True
self.percentage_required = 0.9
self.machine_readable = False
self.measurement_id = None
self.display_probes = False
self.ipv4 = False
self.private = False
self.port = 80
self.size = 64
self.spread = None
# Tags
self.exclude = None
self.include = None
def usage(self, msg=None):
if msg:
print(msg, file=sys.stderr)
print("""General options are:
--verbose or -v : makes the program more talkative
--help or -h : this message
--displayprobes or -o : display the probes numbers (WARNING: big lists)
--country=2LETTERSCODE or -c 2LETTERSCODE : limits the measurements to one country (default is world-wide)
--area=AREACODE or -a AREACODE : limits the measurements to one area such as North-Central (default is world-wide)
--asn=ASnumber or -n ASnumber : limits the measurements to one AS (default is all ASes)
--prefix=IPprefix or -f IPprefix : limits the measurements to one IP prefix (default is all prefixes) WARNING: it must be an *exact* prefix in the global routing table
--probes=N or -s N : selects the probes by giving explicit ID (one ID or a comma-separated list)
--requested=N or -r N : requests N probes (default is %s)
--percentage=X or -p X : stops the program as soon as X %% of the probes reported a result (default is %s %%)
--measurement-ID=N or -m N : do not start a measurement, just analyze a former one
--old_measurement MSMID or -g MSMID : uses the probes of measurement MSMID
--include TAGS or -i TAGS : limits the measurements to probes with these tags (a comma-separated list)
--exclude TAGS or -e TAGS : excludes from measurements the probes with these tags (a comma-separated list)
--port=N or -t N : destination port for TCP (default is %s)
--size=N or -z N : number of bytes in the packet (default is %s bytes)
--ipv4 or -4 : uses IPv4 (default is IPv6, except if the parameter or option is an IP address, then it is automatically found)
--spread or -w : spreads the tests (add a delay before the tests)
--private : makes the measurement private
--machinereadable or -b : machine-readable output, to be consumed by tools like grep or cut
""" % (self.requested, int(self.percentage_required*100), self.port, self.size), file=sys.stderr)
def parse(self, shortOptsSpecific="", longOptsSpecific=[], parseSpecific=None, usage=None):
if usage is None:
usage = self.usage
try:
optlist, args = getopt.getopt (sys.argv[1:],
"4a:bc:e:f:g:hi:m:n:op:r:s:t:vw:z:" + shortOptsSpecific,
["requested=", "country=", "area=", "asn=", "prefix=", "probes=",
"port=", "percentage=", "include=", "exclude=", "version",
"measurement-ID=", "old_measurement=", "displayprobes", "size=",
"ipv4", "private", "machinereadable", "spread=", "verbose", "help"] +
longOptsSpecific)
for option, value in optlist:
if option == "--country" or option == "-c":
self.country = value
elif option == "--area" or option == "-a":
self.area = value
elif option == "--asn" or option == "-n":
self.asn = value
elif option == "--prefix" or option == "-f":
self.prefix = value
elif option == "--probes" or option == "-s":
self.probes = value # Splitting (and syntax checking...) delegated to Atlas
elif option == "--percentage" or option == "-p":
self.percentage_required = float(value)
elif option == "--requested" or option == "-r":
self.requested = int(value)
self.default_requested = False
elif option == "--port" or option == "-t":
self.port = int(value)
elif option == "--measurement-ID" or option == "-m":
self.measurement_id = value
elif option == "--old_measurement" or option == "-g":
self.old_measurement = value
elif option == "--verbose" or option == "-v":
self.verbose = True
elif option == "--ipv4" or option == "-4":
self.ipv4 = True
elif option == "--private":
self.private = True
elif option == "--size" or option == "-z":
self.size = int(value)
elif option == "--spread" or option == "-w":
self.spread = int(value)
elif option == "--displayprobes" or option == "-o":
self.display_probes = True
elif option == "--exclude" or option == "-e":
self.exclude = value.split(",")
elif option == "--include" or option == "-i":
# TODO allows to specify stable probes https://labs.ripe.net/Members/chris_amin/new-ripe-atlas-probe-stability-system-tags
self.include = value.split(",")
elif option == "--machinereadable" or option == "-b":
self.machine_readable = True
elif option == "--help" or option == "-h":
usage()
sys.exit(0)
elif option == "--version":
print("Blaeu version %s" % VERSION)
sys.exit(0)
else:
parseResult = parseSpecific(self, option, value)
if not parseResult:
usage("Unknown option %s" % option)
sys.exit(1)
except getopt.error as reason:
usage(reason)
sys.exit(1)
if self.country is not None:
if self.asn is not None or self.area is not None or self.prefix is not None or \
self.probes is not None:
usage("Specify country *or* area *or* ASn *or* prefix *or* the list of probes")
sys.exit(1)
elif self.area is not None:
if self.asn is not None or self.country is not None or self.prefix is not None or \
self.probes is not None:
usage("Specify country *or* area *or* ASn *or* prefix *or* the list of probes")
sys.exit(1)
elif self.asn is not None:
if self.area is not None or self.country is not None or self.prefix is not None or \
self.probes is not None:
usage("Specify country *or* area *or* ASn *or* prefix *or* the list of probes")
sys.exit(1)
elif self.probes is not None:
if self.country is not None or self.area is not None or self.asn or \
self.prefix is not None:
usage("Specify country *or* area *or* ASn *or* prefix *or* the list of probes")
sys.exit(1)
elif self.prefix is not None:
if self.country is not None or self.area is not None or self.asn or \
self.probes is not None:
usage("Specify country *or* area *or* ASn *or* prefix *or* the list of probes")
sys.exit(1)
if self.probes is not None or self.old_measurement is not None:
if not self.default_requested:
print("Warning: --requested=%d ignored since a list of probes was requested" % self.requested, file=sys.stderr)
if self.old_measurement is not None:
if self.country is not None:
print("Warning: --country ignored since we use probes from a previous measurement", file=sys.stderr)
if self.area is not None:
print("Warning: --area ignored since we use probes from a previous measurement", file=sys.stderr)
if self.prefix is not None:
print("Warning: --prefix ignored since we use probes from a previous measurement", file=sys.stderr)
if self.asn is not None:
print("Warning: --asn ignored since we use probes from a previous measurement", file=sys.stderr)
if self.probes is not None:
print("Warning: --probes ignored since we use probes from a previous measurement", file=sys.stderr)
# TODO include and exclude should trigger a similar warning...
if self.probes is not None:
self.requested = len(self.probes.split(","))
data = { "is_oneoff": True,
"definitions": [
{"description": "", "port": self.port} ],
"probes": [
{"requested": self.requested} ] }
if self.old_measurement is not None:
data["probes"][0]["requested"] = 500 # Dummy value, anyway,
# but necessary to get
# all the probes
# TODO: the huge value of "requested" makes us wait a very long time
data["probes"][0]["type"] = "msm"
data["probes"][0]["value"] = self.old_measurement
data["definitions"][0]["description"] += (" from probes of measurement #%s" % self.old_measurement)
else:
if self.probes is not None:
data["probes"][0]["type"] = "probes"
data["probes"][0]["value"] = self.probes
else:
if self.country is not None:
data["probes"][0]["type"] = "country"
data["probes"][0]["value"] = self.country
data["definitions"][0]["description"] += (" from %s" % self.country)
elif self.area is not None:
data["probes"][0]["type"] = "area"
data["probes"][0]["value"] = self.area
data["definitions"][0]["description"] += (" from %s" % self.area)
elif self.asn is not None:
data["probes"][0]["type"] = "asn"
data["probes"][0]["value"] = self.asn
data["definitions"][0]["description"] += (" from AS #%s" % self.asn)
elif self.prefix is not None:
data["probes"][0]["type"] = "prefix"
data["probes"][0]["value"] = self.prefix
data["definitions"][0]["description"] += (" from prefix %s" % self.prefix)
else:
data["probes"][0]["type"] = "area"
data["probes"][0]["value"] = "WW"
if self.ipv4:
data["definitions"][0]['af'] = 4
else:
data["definitions"][0]['af'] = 6
if self.private:
data["definitions"][0]['is_public'] = False
if self.size is not None:
data["definitions"][0]['size'] = self.size
if self.spread is not None:
data["definitions"][0]['spread'] = self.spread
data["probes"][0]["tags"] = {}
if self.include is not None:
data["probes"][0]["tags"]["include"] = copy.copy(self.include)
else:
data["probes"][0]["tags"]["include"] = []
if self.ipv4:
data["probes"][0]["tags"]["include"].append("system-ipv4-works") # Some probes cannot do ICMP outgoing (firewall?)
else:
data["probes"][0]["tags"]["include"].append("system-ipv6-works")
if self.exclude is not None:
data["probes"][0]["tags"]["exclude"] = copy.copy(self.exclude)
if self.verbose:
print("Blaeu version %s" % VERSION)
return args, data
class JsonRequest(urllib.request.Request):
def __init__(self, url):
urllib.request.Request.__init__(self, url)
self.url = url
self.add_header("Content-Type", "application/json")
self.add_header("Accept", "application/json")
self.add_header("User-Agent", "RIPEAtlas.py")
def __str__(self):
return self.url
class Measurement():
""" An Atlas measurement, identified by its ID (such as #1010569) in the field "id" """
def __init__(self, data, wait=True, sleep_notification=None, key=None, id=None):
"""
Creates a measurement."data" must be a dictionary (*not* a JSON string) having the members
requested by the Atlas documentation. "wait" should be set to False for periodic (not
oneoff) measurements. "sleep_notification" is a lambda taking one parameter, the
sleep delay: when the module has to sleep, it calls this lambda, allowing you to be informed of
the delay. "key" is the API key. If None, it will be read in the configuration file.
If "data" is None and id is not, a dummy measurement will be created, mapped to
the existing measurement having this ID.
"""
if data is None and id is None:
raise RequestSubmissionError("No data and no measurement ID")
# TODO: when creating a dummy measurement, a key may not be necessary if the measurement is public
if not key:
if os.environ.get("ATLASAUTH"):
# use envvar ATLASAUTH for the key
key = os.environ.get("ATLASAUTH")
else:
# use file for key if envvar ATLASAUTH isn't set
if not os.path.exists(authfile):
raise AuthFileNotFound("Authentication file %s not found" % authfile)
auth = open(authfile)
key = auth.readline()
if key is None or key == "":
raise AuthFileEmpty("Authentication file %s empty or missing a end-of-line at the end" % authfile)
key = key.rstrip('\n')
auth.close()
self.url = base_url + "/?key=%s" % key
self.url_probes = base_url + "/%s/?fields=probes,status" + "&key=%s" % key
self.url_status = base_url + "/%s/?fields=status" + "&key=%s" % key
self.url_results = base_url + "/%s/results/" + "?key=%s" % key
self.url_all = base_url + "/%s/" + "?key=%s" % key
self.url_latest = base_url + "-latest/%s/?versions=%s"
self.status = None
if data is not None:
self.json_data = json.dumps(data).encode('utf-8')
self.notification = sleep_notification
request = JsonRequest(self.url)
try:
# Start the measurement
conn = urllib.request.urlopen(request, self.json_data)
# Now, parse the answer
results = json.loads(conn.read().decode('utf-8'))
self.id = results["measurements"][0]
conn.close()
except urllib.error.HTTPError as e:
raise RequestSubmissionError("Status %s, reason \"%s : %s\"" % \
(e.code, e.reason, e.read()))
except urllib.error.URLError as e:
raise RequestSubmissionError("Reason \"%s\"" % \
(e.reason))
self.gen = random.Random()
self.time = time.gmtime()
if not wait:
return
# Find out how many probes were actually allocated to this measurement
enough = False
left = 30 # Maximum number of tests
requested = data["probes"][0]["requested"]
fields_delay = fields_delay_base + (requested * fields_delay_factor)
while not enough:
# Let's be patient
if self.notification is not None:
self.notification(fields_delay)
time.sleep(fields_delay)
fields_delay *= 2
try:
request = JsonRequest((self.url_probes % self.id) + \
("&defeatcaching=dc%s" % self.gen.randint(1,10000))) # A random
# component is necesary to defeat caching (even Cache-Control sems ignored)
conn = urllib.request.urlopen(request)
# Now, parse the answer
meta = json.loads(conn.read().decode('utf-8'))
self.status = meta["status"]["name"]
if meta["status"]["name"] == "Specified" or \
meta["status"]["name"] == "Scheduled":
# Not done, loop
left -= 1
if left <= 0:
raise FieldsQueryError("Maximum number of status queries reached")
elif meta["status"]["name"] == "Ongoing":
enough = True
self.num_probes = len(meta["probes"])
else:
raise InternalError("Internal error in #%s, unexpected status when querying the measurement fields: \"%s\"" % (self.id, meta["status"]))
conn.close()
except urllib.error.URLError as e:
raise FieldsQueryError("%s" % e.reason)
else:
self.id = id
self.notification = None
try:
conn = urllib.request.urlopen(JsonRequest(self.url_status % self.id))
except urllib.error.HTTPError as e:
if e.code == 404:
raise MeasurementNotFound
else:
raise MeasurementAccessError("HTTP %s, %s %s" % (e.code, e.reason, e.read()))
except urllib.error.URLError as e:
raise MeasurementAccessError("Reason \"%s\"" % \
(e.reason))
result_status = json.loads(conn.read().decode('utf-8'))
status = result_status["status"]["name"]
self.status = status
if status != "Ongoing" and status != "Stopped":
raise MeasurementAccessError("Invalid status \"%s\"" % status)
try:
conn = urllib.request.urlopen(JsonRequest(self.url_probes % self.id))
except urllib.error.HTTPError as e:
if e.code == 404:
raise MeasurementNotFound
else:
raise MeasurementAccessError("%s %s" % (e.reason, e.read()))
except urllib.error.URLError as e:
raise MeasurementAccessError("Reason \"%s\"" % \
(e.reason))
result_status = json.loads(conn.read().decode('utf-8'))
self.num_probes = len(result_status["probes"])
try:
conn = urllib.request.urlopen(JsonRequest(self.url_all % self.id))
except urllib.error.HTTPError as e:
if e.code == 404:
raise MeasurementNotFound
else:
raise MeasurementAccessError("%s %s" % (e.reason, e.read()))
except urllib.error.URLError as e:
raise MeasurementAccessError("Reason \"%s\"" % \
(e.reason))
result_status = json.loads(conn.read().decode('utf-8'))
self.time = time.gmtime(result_status["start_time"])
self.description = result_status["description"]
self.interval = result_status["interval"]
def results(self, wait=True, percentage_required=0.9, latest=None):
"""Retrieves the result. "wait" indicates if you are willing to wait until
the measurement is over (otherwise, you'll get partial
results). "percentage_required" is meaningful only when you wait
and it indicates the percentage of the allocated probes that
have to report before the function returns (warning: the
measurement may stop even if not enough probes reported so you
always have to check the actual number of reporting probes in
the result). "latest" indicates that you want to retrieve only
the last N results (by default, you get all the results).
"""
if latest is not None:
wait = False
if latest is None:
request = JsonRequest(self.url_results % self.id)
else:
request = JsonRequest(self.url_latest% (self.id, latest))
if wait:
enough = False
attempts = 0
results_delay = results_delay_base + (self.num_probes * results_delay_factor)
maximum_time_for_results = maximum_time_for_results_base + \
(self.num_probes * maximum_time_for_results_factor)
start = time.time()
elapsed = 0
result_data = None
while not enough and elapsed < maximum_time_for_results:
if self.notification is not None:
self.notification(results_delay)
time.sleep(results_delay)
results_delay *= 2
attempts += 1
elapsed = time.time() - start
try:
conn = urllib.request.urlopen(request)
result_data = json.loads(conn.read().decode('utf-8'))
num_results = len(result_data)
if num_results >= self.num_probes*percentage_required:
# Requesting a strict equality may be too
# strict: if an allocated probe does not
# respond, we will have to wait for the stop
# of the measurement (many minutes). Anyway,
# there is also the problem that a probe may
# have sent only a part of its measurements.
enough = True
else:
conn = urllib.request.urlopen(JsonRequest(self.url_status % self.id))
result_status = json.loads(conn.read().decode('utf-8'))
status = result_status["status"]["name"]
if status == "Ongoing":
# Wait a bit more
pass
elif status == "Stopped":
enough = True # Even if not enough probes
else:
raise InternalError("Unexpected status when retrieving the measurement: \"%s\"" % \
result_data["status"])
conn.close()
except urllib.error.HTTPError as e:
if e.code != 404: # Yes, we may have no result file at
# all for some time
raise ResultError(str(e.code) + " " + e.reason + " " + str(e.read()))
except urllib.error.URLError as e:
raise ResultError("Reason \"%s\"" % \
(e.reason))
if result_data is None:
raise ResultError("No results retrieved")
else:
try:
conn = urllib.request.urlopen(request)
result_data = json.loads(conn.read().decode('utf-8'))
except urllib.error.URLError as e:
raise ResultError(e.reason)
return result_data
././@PaxHeader 0000000 0000000 0000000 00000000026 00000000000 010213 x ustar 00 22 mtime=1522162031.0
blaeu-1.1.10/LICENCE 0000644 0001750 0001750 00000002425 13256454557 015120 0 ustar 00bortzmeyer bortzmeyer Copyright (c) 2017, Stephane Bortzmeyer
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
././@PaxHeader 0000000 0000000 0000000 00000000026 00000000000 010213 x ustar 00 22 mtime=1522162031.0
blaeu-1.1.10/MANIFEST.in 0000644 0001750 0001750 00000000020 13256454557 015656 0 ustar 00bortzmeyer bortzmeyer include LICENCE
././@PaxHeader 0000000 0000000 0000000 00000000033 00000000000 010211 x ustar 00 27 mtime=1706190263.134136
blaeu-1.1.10/PKG-INFO 0000644 0001750 0001750 00000005767 14554462667 015247 0 ustar 00bortzmeyer bortzmeyer Metadata-Version: 2.1
Name: blaeu
Version: 1.1.10
Summary: Tools to create (and analyze) RIPE Atlas network measurements
Home-page: https://framagit.org/bortzmeyer/blaeu
Author: Stéphane Bortzmeyer
Author-email: stephane+frama@bortzmeyer.org
License: BSD
Keywords: networking atlas monitoring ip ping traceroute dig
Classifier: Development Status :: 4 - Beta
Classifier: Intended Audience :: System Administrators
Classifier: Intended Audience :: Telecommunications Industry
Classifier: Topic :: System :: Networking
Classifier: License :: OSI Approved :: MIT License
Classifier: Programming Language :: Python :: 3
Requires-Python: >=3
Provides-Extra: dev
License-File: LICENCE
Blaeu, creating measurements on RIPE Atlas probes
=================================================
This is a set of `Python `__ programs to start
distributed Internet measurements on the network of `RIPE Atlas
probes `__, and to analyze their results.
For installation, you can use usual Python tools, for instance:
::
pip3 install blaeu
(On a Debian machine, the prerequitises are packages python3-pip,
python3-openssl, python3-dnspython, and python3-cymruwhois. This is only
if you install manually, otherwise pip3 will install the dependencies.)
Usage requires a RIPE Atlas API key (which itself requires a RIPE
account), and RIPE Atlas credits. If you don’t have a RIPE account,
`register first `__. Once you have an account,
`create a key `__, grant it the right to
``schedule a new measurement``, and put the key in ``~/.atlas/auth``. If
you don’t have Atlas credits, host a probe,or become a
`LIR `__ or ask a
friend.
You can then use the four programs (``-h`` will give you a complete list
of their options):
- ``blaeu-reach target-IP-address`` (test reachability of the target,
like ``ping``)
- ``blaeu-traceroute target-IP-address`` (like ``traceroute``)
- ``blaeu-resolve name`` (use the DNS to resolve the name)
- ``blaeu-cert name`` (display the PKIX certificate)
You may also be interested by `my article at RIPE
Labs `__.
Blaeu requires Python 3.
Note that `the old
version `__
ran on Python 2 but is no longer maintained. (It was `partially
documented at RIPE
Labs `__.)
Name
----
It comes from the `famous Dutch
cartographer `__. The logo
of the project comes from his “Theatrum Orbis Terrarum” (see `the
source `__).
Reference site
--------------
`On FramaGit `__
Author
------
Stéphane Bortzmeyer stephane+frama@bortzmeyer.org
././@PaxHeader 0000000 0000000 0000000 00000000026 00000000000 010213 x ustar 00 22 mtime=1681820889.0
blaeu-1.1.10/README.rst 0000644 0001750 0001750 00000004516 14417506331 015611 0 ustar 00bortzmeyer bortzmeyer Blaeu, creating measurements on RIPE Atlas probes
=================================================
This is a set of `Python `__ programs to start
distributed Internet measurements on the network of `RIPE Atlas
probes `__, and to analyze their results.
For installation, you can use usual Python tools, for instance:
::
pip3 install blaeu
(On a Debian machine, the prerequitises are packages python3-pip,
python3-openssl, python3-dnspython, and python3-cymruwhois. This is only
if you install manually, otherwise pip3 will install the dependencies.)
Usage requires a RIPE Atlas API key (which itself requires a RIPE
account), and RIPE Atlas credits. If you don’t have a RIPE account,
`register first `__. Once you have an account,
`create a key `__, grant it the right to
``schedule a new measurement``, and put the key in ``~/.atlas/auth``. If
you don’t have Atlas credits, host a probe,or become a
`LIR `__ or ask a
friend.
You can then use the four programs (``-h`` will give you a complete list
of their options):
- ``blaeu-reach target-IP-address`` (test reachability of the target,
like ``ping``)
- ``blaeu-traceroute target-IP-address`` (like ``traceroute``)
- ``blaeu-resolve name`` (use the DNS to resolve the name)
- ``blaeu-cert name`` (display the PKIX certificate)
You may also be interested by `my article at RIPE
Labs `__.
Blaeu requires Python 3.
Note that `the old
version `__
ran on Python 2 but is no longer maintained. (It was `partially
documented at RIPE
Labs `__.)
Name
----
It comes from the `famous Dutch
cartographer `__. The logo
of the project comes from his “Theatrum Orbis Terrarum” (see `the
source `__).
Reference site
--------------
`On FramaGit `__
Author
------
Stéphane Bortzmeyer stephane+frama@bortzmeyer.org
././@PaxHeader 0000000 0000000 0000000 00000000026 00000000000 010213 x ustar 00 22 mtime=1663079384.0
blaeu-1.1.10/blaeu-cert 0000755 0001750 0001750 00000015256 14310111730 016061 0 ustar 00bortzmeyer bortzmeyer #!/usr/bin/env python3
# -*- coding: utf-8 -*-
""" Python code to start a RIPE Atlas UDM (User-Defined
Measurement). This one is to test X.509/PKIX certificates in TLS servers.
You'll need an API key in ~/.atlas/auth.
After launching the measurement, it downloads the results and analyzes
them, displaying the name ("subject" in X.509 parlance) or issuer.
Stéphane Bortzmeyer
"""
import json
import time
import os
import string
import re
import sys
import time
import socket
import collections
import copy
import Blaeu
# https://github.com/pyca/pyopenssl https://pyopenssl.readthedocs.org/en/stable/
import OpenSSL.crypto
import cryptography
config = Blaeu.Config()
# Default values
config.display = "n" #Name
config.sni = True
config.resolve_on_probe = False
# Override what's in the Blaeu package
config.port = 443
class Set():
def __init__(self):
self.total = 0
def usage(msg=None):
print("Usage: %s target-name-or-IP" % sys.argv[0], file=sys.stderr)
config.usage(msg)
print("""Also:
--issuer or -I : displays the issuer (default is to display the name)
--key or -k : displays the public key (default is to display the name)
--serial or -S : displays the serial number (default is to display the name)
--expiration or -E : displays the expiration datetime (default is to display the name)
--no-sni : do not send the SNI (Server Name Indication) (default is to send it)
--resolve-on-probe : resolve domain names on the probe (default is to use RIPE resolvers)""", file=sys.stderr)
def format_name(n):
result = ""
components = n.get_components()
for (k, v) in components:
result += "/%s=%s" % (k.decode(), v.decode())
return result
def specificParse(config, option, value):
result = True
if option == "--issuer" or option == "-I":
config.display = "i"
elif option == "--key" or option == "-k":
config.display = "k"
elif option == "--serial" or option == "-S":
config.display = "s"
elif option == "--expiration" or option == "-E":
config.display = "e"
elif option == "--no-sni":
config.sni = False
elif option == "--resolve-on-probe":
config.resolve_on_probe = True
else:
result = False
return result
(args, data) = config.parse("IkSE", ["issuer", "serial", "expiration", "key", "no-sni", "resolve-on-probe"],
specificParse, usage)
if len(args) != 1:
usage("Not the good number of arguments")
sys.exit(1)
target = args[0]
if config.measurement_id is None:
data["definitions"][0]["target"] = target
data["definitions"][0]["type"] = "sslcert"
data["definitions"][0]["description"] = "X.509 cert of %s" % target
del data["definitions"][0]["size"] # Meaningless argument
if target.find(':') > -1: # TODO: or use is_ip_address(str) from blaeu-reach?
config.ipv4 = False
af = 6
if config.include is not None:
data["probes"][0]["tags"]["include"] = copy.copy(config.include)
data["probes"][0]["tags"]["include"].append("system-ipv6-works")
else:
data["probes"][0]["tags"]["include"] = ["system-ipv6-works",]
elif re.match("^[0-9.]+$", target):
config.ipv4 = True
af = 4
if config.include is not None:
data["probes"][0]["tags"]["include"] = copy.copy(config.include)
data["probes"][0]["tags"]["include"].append("system-ipv4-works")
else:
data["probes"][0]["tags"]["include"] = ["system-ipv4-works",]
else:
# Hostname
if config.ipv4:
af = 4
else:
af = 6
data["definitions"][0]['af'] = af
if config.sni:
data["definitions"][0]['hostname'] = target
if config.resolve_on_probe:
data["definitions"][0]['resolve_on_probe'] = True
if config.verbose:
print(data)
measurement = Blaeu.Measurement(data)
if config.verbose:
print("Measurement #%s to %s uses %i probes" % (measurement.id, target,
measurement.num_probes))
rdata = measurement.results(wait=True, percentage_required=config.percentage_required)
else:
measurement = Blaeu.Measurement(data=None, id=config.measurement_id)
rdata = measurement.results(wait=False)
sets = collections.defaultdict(Set)
if config.display_probes:
probes_sets = collections.defaultdict(Set)
print(("%s probes reported" % len(rdata)))
for result in rdata:
if config.display_probes:
probe_id = result["prb_id"]
if 'cert' in result:
# TODO: handle chains of certificates
x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, str(result['cert'][0]))
detail = ""
content = format_name(x509.get_subject())
if config.display == "i":
content = format_name(x509.get_issuer())
elif config.display == "k":
key = x509.get_pubkey()
content = "%s, type %s, %s bits" % \
(key.to_cryptography_key().public_bytes(cryptography.hazmat.primitives.serialization.Encoding.PEM, cryptography.hazmat.primitives.serialization.PublicFormat.SubjectPublicKeyInfo).decode().replace("\n", " ").replace("-----BEGIN PUBLIC KEY----- ", "")[:80] + "...",
key.type(), key.bits())
elif config.display == "s":
content = format(x509.get_serial_number(), '05x')
elif config.display == "e":
if x509.has_expired():
detail = " (EXPIRED)"
# TODO: better format of the date?
content = "%s%s" % (x509.get_notAfter().decode(), detail)
value = "%s%s" % (content, detail)
else:
if 'err' in result:
error = result['err']
elif 'alert' in result:
error = result['alert']
else:
error = "UNKNOWN ERROR"
value = "FAILED TO GET A CERT: %s" % error
sets[value].total += 1
if config.display_probes:
if value in probes_sets:
probes_sets[value].append(probe_id)
else:
probes_sets[value] = [probe_id,]
sets_data = sorted(sets, key=lambda s: sets[s].total, reverse=False)
for myset in sets_data:
detail = ""
if config.display_probes:
detail = "(probes %s)" % probes_sets[myset]
print("[%s] : %i occurrences %s" % (myset, sets[myset].total, detail))
print(("Test #%s done at %s" % (measurement.id,
time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))))
././@PaxHeader 0000000 0000000 0000000 00000000026 00000000000 010213 x ustar 00 22 mtime=1663052866.0
blaeu-1.1.10/blaeu-reach 0000755 0001750 0001750 00000016346 14310026102 016205 0 ustar 00bortzmeyer bortzmeyer #!/usr/bin/env python3
# -*- coding: utf-8 -*-
""" Python code to start a RIPE Atlas UDM (User-Defined
Measurement). This one is for running IPv4 or IPv6 ICMP queries to
test reachability.
You'll need an API key in ~/.atlas/auth.
After launching the measurement, it downloads the results and analyzes
them.
Stéphane Bortzmeyer
"""
import json
import time
import os
import sys
import time
import socket
import copy
import collections
import Blaeu
config = Blaeu.Config()
# Default values
config.tests = 3 # ICMP packets per probe
config.by_probe = False # Default is to count by test, not by probe
config.display_probes = False
class Set():
def __init__(self):
self.failed = True
def is_ip_address(str):
try:
addr = socket.inet_pton(socket.AF_INET6, str)
except socket.error: # not a valid IPv6 address
try:
addr = socket.inet_pton(socket.AF_INET, str)
except socket.error: # not a valid IPv4 address either
return False
return True
def usage(msg=None):
print("Usage: %s target-IP-address ..." % sys.argv[0], file=sys.stderr)
config.usage(msg)
print("""Also:
--tests=N or -d N : send N ICMP packets from each probe (default is %s)
--by_probe : count the percentage of success by probe, not by test (useless if --tests=1)
""" % (config.tests), file=sys.stderr)
def specificParse(config, option, value):
result = True
if option == "--tests" or option == "-d":
config.tests = int(value)
elif option == "--by_probe":
config.by_probe = True
else:
result = False
return result
args, data = config.parse("d:", ["by_probe", "tests="], specificParse,
usage)
targets = args
if len(targets) == 0:
usage("No target found")
sys.exit(1)
if config.verbose and config.machine_readable:
usage("Specify verbose *or* machine-readable output")
sys.exit(1)
if config.display_probes and config.machine_readable:
usage("Display probes *or* machine-readable output")
sys.exit(1)
data["definitions"][0]["type"] = "ping"
del data["definitions"][0]["port"]
data["definitions"][0]["packets"] = config.tests
for target in targets:
if not is_ip_address(target):
print(("Target must be an IP address, NOT AN HOST NAME"), file=sys.stderr)
sys.exit(1)
data["definitions"][0]["target"] = target
data["definitions"][0]["description"] = ("Ping %s" % target) + data["definitions"][0]["description"]
if target.find(':') > -1:
config.ipv4 = False
data["definitions"][0]['af'] = 6
else:
config.ipv4 = True
data["definitions"][0]['af'] = 4
# Yes, it was already done in parse() but we have to do it again now that we
# know the address family of the target. See bug #9. Note that we silently
# override a possible explicit choice of the user (her -4 may be ignored).
if config.include is not None:
data["probes"][0]["tags"]["include"] = copy.copy(config.include)
else:
data["probes"][0]["tags"]["include"] = []
if config.ipv4:
data["probes"][0]["tags"]["include"].append("system-ipv4-works") # Some probes cannot do ICMP outgoing (firewall?)
else:
data["probes"][0]["tags"]["include"].append("system-ipv6-works")
if config.exclude is not None:
data["probes"][0]["tags"]["exclude"] = copy.copy(config.exclude)
if config.measurement_id is None:
if config.verbose:
print(data)
measurement = Blaeu.Measurement(data)
if config.old_measurement is None:
config.old_measurement = measurement.id
if config.verbose:
print("Measurement #%s to %s uses %i probes" % (measurement.id, target,
measurement.num_probes))
# Retrieve the results
rdata = measurement.results(wait=True, percentage_required=config.percentage_required)
else:
measurement = Blaeu.Measurement(data=None, id=config.measurement_id)
rdata = measurement.results(wait=False)
if config.verbose:
print("%i results from already-done measurement #%s" % (len(rdata), measurement.id))
if len(rdata) == 0:
print("Warning: zero results. Measurement not terminated? May be retry later with --measurement-ID=%s ?" % measurement.id, file=sys.stderr)
total_rtt = 0
num_rtt = 0
num_error = 0
num_timeout = 0
num_tests = 0
if config.by_probe:
probes_success = 0
probes_failure = 0
num_probes = 0
if not config.machine_readable and config.measurement_id is None:
print(("%s probes reported" % len(rdata)))
if config.display_probes:
failed_probes = collections.defaultdict(Set)
for result in rdata:
probe_ok = False
probe = result["prb_id"]
if config.by_probe:
num_probes += 1
for test in result["result"]:
num_tests += 1
if "rtt" in test:
total_rtt += int(test["rtt"])
num_rtt += 1
probe_ok = True
elif "error" in test:
num_error += 1
elif "x" in test:
num_timeout += 1
else:
print(("Result has no field rtt, or x or error"), file=sys.stderr)
sys.exit(1)
if config.by_probe:
if probe_ok:
probes_success += 1
else:
probes_failure += 1
if config.display_probes and not probe_ok:
failed_probes[probe].failed = True
if not config.machine_readable:
print(("Test #%s done at %s" % (measurement.id, time.strftime("%Y-%m-%dT%H:%M:%SZ", measurement.time))))
if num_rtt == 0:
if not config.machine_readable:
print("No successful test")
else:
if not config.machine_readable:
if not config.by_probe:
print(("Tests: %i successful tests (%.1f %%), %i errors (%.1f %%), %i timeouts (%.1f %%), average RTT: %i ms" % \
(num_rtt, num_rtt*100.0/num_tests,
num_error, num_error*100.0/num_tests,
num_timeout, num_timeout*100.0/num_tests, total_rtt/num_rtt)))
else:
print(("Tests: %i successful probes (%.1f %%), %i failed (%.1f %%), average RTT: %i ms" % \
(probes_success, probes_success*100.0/num_probes,
probes_failure, probes_failure*100.0/num_probes,
total_rtt/num_rtt)))
if len(targets) > 1 and not config.machine_readable:
print("")
if config.display_probes:
all = list(failed_probes.keys())
if all != []:
print(all)
if config.machine_readable:
if num_rtt != 0:
percent_rtt = total_rtt/num_rtt
else:
percent_rtt = 0
print(",".join([target, str(measurement.id), "%s/%s" % (len(rdata),measurement.num_probes), \
time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "%i" % num_rtt, \
"%.1f" % (num_rtt*100.0/num_tests), "%i" % num_error, "%.1f" % (num_error*100.0/num_tests), \
"%i" % num_timeout, "%.1f" % (num_timeout*100.0/num_tests), "%i" % (percent_rtt)]))
././@PaxHeader 0000000 0000000 0000000 00000000026 00000000000 010213 x ustar 00 22 mtime=1705913815.0
blaeu-1.1.10/blaeu-resolve 0000755 0001750 0001750 00000047475 14553426727 016643 0 ustar 00bortzmeyer bortzmeyer #!/usr/bin/env python3
# -*- coding: utf-8 -*-
""" Python code to start a RIPE Atlas UDM (User-Defined
Measurement). This one is for running DNS to resolve a name from many
places, in order to survey local cache poisonings, effect of
hijackings and other DNS rejuvenation effects.
You'll need an API key in ~/.atlas/auth.
After launching the measurement, it downloads the results and analyzes
them.
Stéphane Bortzmeyer
"""
import json
import sys
import time
import base64
import re
import copy
import collections
# DNS Python http://www.dnspython.org/
import dns.message
import Blaeu
config = Blaeu.Config()
# Default values
config.qtype = 'AAAA'
config.qclass = "IN"
config.display_resolvers = False
config.display_rtt = False
config.display_validation = False
config.edns_size = None
config.dnssec = False
config.dnssec_checking = True
config.nameserver = None
config.recursive = True
config.sort = False
config.nsid = False
config.only_one_per_probe = True
config.protocol = "UDP"
config.tls = False
config.probe_id = False
config.answer_section = True
config.authority_section = False
config.additional_section = False
# Local values
edns_size = None
# Constants
MAXLEN = 80 # Maximum length of a displayed resource record
class Set():
def __init__(self):
self.total = 0
self.successes = 0
self.rtt = 0
def usage(msg=None):
print("Usage: %s domain-name" % sys.argv[0], file=sys.stderr)
config.usage(msg)
print("""Also:
--displayresolvers or -l : display the resolvers IP addresses (WARNING: big lists)
--norecursive or -Z : asks the resolver to NOT recurse (default is to recurse, note --norecursive works ONLY if asking a specific resolver, not with the default one)
--dnssec or -D : asks the resolver the DNSSEC records
--nsid : asks the resolver with NSID (name server identification)
--ednssize=N or -B N : asks for EDNS with the "payload size" option (default is very old DNS, without EDNS)
--tcp: uses TCP (default is UDP)
--tls: uses TLS (implies TCP)
--checkingdisabled or -k : asks the resolver to NOT perform DNSSEC validation
--displayvalidation or -j : displays the DNSSEC validation status
--displayrtt : displays the average RTT
--authority : displays the Authority section of the answer
--additional : displays the Additional section of the answer
--sort or -S : sort the result sets
--type or -q : query type (default is %s)
--class : query class (default is %s)
--severalperprobe : count all the resolvers of each probe (default is to count only the first to reply)
--nameserver=name_or_IPaddr[,...] or -x name_or_IPaddr : query this name server (default is to query the probe's resolver)
--probe_id : prepend probe ID (and timestamp) to the domain name (default is to abstain)
""" % (config.qtype, config.qclass), file=sys.stderr)
def specificParse(config, option, value):
result = True
if option == "--type" or option == "-q":
config.qtype = value
elif option == "--class": # For Chaos, use "CHAOS", not "CH"
config.qclass = value
elif option == "--norecursive" or option == "-Z":
config.recursive = False
elif option == "--dnssec" or option == "-D":
config.dnssec = True
elif option == "--nsid":
config.nsid = True
elif option == "--probe_id":
config.probe_id = True
elif option == "--ednssize" or option == "-B":
config.edns_size = int(value)
elif option == "--tcp":
config.protocol = "TCP"
elif option == "--tls":
config.tls = True
elif option == "--checkingdisabled" or option == "-k":
config.dnssec_checking = False
elif option == "--sort" or option == "-S":
config.sort = True
elif option == "--authority":
config.answer_section = False
config.authority_section = True
elif option == "--additional":
config.answer_section = False
config.additional_section = True
elif option == "--nameserver" or option == "-x":
config.nameserver = value
config.nameservers = config.nameserver.split(",")
elif option == "--displayresolvers" or option == "-l":
config.display_resolvers = True
elif option == "--displayvalidation" or option == "-j":
config.display_validation = True
elif option == "--displayrtt":
config.display_rtt = True
elif option == "--severalperprobe":
config.only_one_per_probe = False
else:
result = False
return result
args, data = config.parse("q:ZDkSx:ljB:", ["type=", "class=", "ednssize=",
"displayresolvers", "probe_id",
"displayrtt", "displayvalidation",
"dnssec", "nsid", "norecursive",
"authority", "additional",
"tcp", "tls", "checkingdisabled",
"nameserver=", "sort",
"severalperprobe"], specificParse,
usage)
if len(args) != 1:
usage()
sys.exit(1)
domainname = args[0]
if config.tls:
config.protocol = "TCP"
# We don't set the port (853) but Atlas does it for us
data["definitions"][0]["type"] = "dns"
del data["definitions"][0]["size"]
del data["definitions"][0]["port"]
data["definitions"][0]["query_argument"] = domainname
data["definitions"][0]["description"] = ("DNS resolution of %s/%s" % (domainname, config.qtype)) + data["definitions"][0]["description"]
data["definitions"][0]["query_class"] = config.qclass
data["definitions"][0]["query_type"] = config.qtype
if config.edns_size is not None and config.protocol == "UDP":
data["definitions"][0]["udp_payload_size"] = config.edns_size
edns_size = config.edns_size
if config.dnssec or config.display_validation: # https://atlas.ripe.net/docs/api/v2/reference/#!/measurements/Dns_Type_Measurement_List_POST
data["definitions"][0]["set_do_bit"] = True
if config.edns_size is None and config.protocol == "UDP":
edns_size = 4096
if config.nsid:
data["definitions"][0]["set_nsid_bit"] = True
if config.edns_size is None and config.protocol == "UDP":
edns_size = 1024
if edns_size is not None and config.protocol == "UDP":
data["definitions"][0]["udp_payload_size"] = edns_size
if not config.dnssec_checking:
data["definitions"][0]["set_cd_bit"] = True
if config.recursive:
data["definitions"][0]["set_rd_bit"] = True
else:
data["definitions"][0]["set_rd_bit"] = False
if config.tls:
data["definitions"][0]["tls"] = True
if config.probe_id:
data["definitions"][0]["prepend_probe_id"] = True
data["definitions"][0]["protocol"] = config.protocol
if config.verbose and config.machine_readable:
usage("Specify verbose *or* machine-readable output")
sys.exit(1)
if (config.display_probes or config.display_resolvers or config.display_rtt) and config.machine_readable:
usage("Display probes/resolvers/RTT *or* machine-readable output")
sys.exit(1)
if config.nameserver is None:
config.nameservers = [None,]
for nameserver in config.nameservers:
if nameserver is None:
data["definitions"][0]["use_probe_resolver"] = True
# Exclude probes which do not have at least one working resolver
data["probes"][0]["tags"]["include"].append("system-resolves-a-correctly")
data["probes"][0]["tags"]["include"].append("system-resolves-aaaa-correctly")
else:
data["definitions"][0]["use_probe_resolver"] = False
data["definitions"][0]["target"] = nameserver
data["definitions"][0]["description"] += (" via nameserver %s" % nameserver) # TODO if several nameservers, they addresses are added after each other :-(
if nameserver.find(':') > -1: # TODO: or use is_ip_address(str) from blaeu-reach?
config.ipv4 = False
data["definitions"][0]['af'] = 6
if config.include is not None:
data["probes"][0]["tags"]["include"] = copy.copy(config.include)
data["probes"][0]["tags"]["include"].append("system-ipv6-works")
else:
data["probes"][0]["tags"]["include"] = ["system-ipv6-works",]
elif re.match("^[0-9.]+$", nameserver):
config.ipv4 = True
data["definitions"][0]['af'] = 4
if config.include is not None:
data["probes"][0]["tags"]["include"] = copy.copy(config.include)
data["probes"][0]["tags"]["include"].append("system-ipv4-works")
else:
data["probes"][0]["tags"]["include"] = ["system-ipv4-works",]
else: # Probably an host name
pass
if config.measurement_id is None:
if config.verbose:
print(data)
measurement = Blaeu.Measurement(data,
lambda delay: sys.stderr.write(
"Sleeping %i seconds...\n" % delay))
if not config.machine_readable and config.verbose:
print("Measurement #%s for %s/%s uses %i probes" % \
(measurement.id, domainname, config.qtype, measurement.num_probes))
old_measurement = measurement.id
results = measurement.results(wait=True)
else:
measurement = Blaeu.Measurement(data=None, id=config.measurement_id)
results = measurement.results(wait=False)
if config.verbose:
print("%i results from already-done measurement %s" % (len(results), measurement.id))
if len(results) == 0:
print("Warning: zero results. Measurement not terminated? May be retry later with --measurement-ID=%s ?" % (measurement.id), file=sys.stderr)
probes = 0
successes = 0
qtype_num = dns.rdatatype.from_text(config.qtype) # Raises dns.rdatatype.UnknownRdatatype if unknown
sets = collections.defaultdict(Set)
if config.display_probes:
probes_sets = collections.defaultdict(Set)
if config.display_resolvers:
resolvers_sets = collections.defaultdict(Set)
for result in results:
probes += 1
probe_id = result["prb_id"]
first_error = ""
probe_resolves = False
resolver_responds = False
all_timeout = True
if "result" in result:
result_set = [{'result': result['result']},]
elif "resultset" in result:
result_set = result['resultset']
elif "error" in result:
result_set = []
myset = []
if "timeout" in result['error']:
myset.append("TIMEOUT")
elif "socket" in result['error']:
all_timeout = False
myset.append("NETWORK PROBLEM WITH RESOLVER")
elif "TUCONNECT" in result['error']:
all_timeout = False
myset.append("TUCONNECT (may be a TLS negotiation error or a TCP connection issue)")
else:
all_timeout = False
myset.append("NO RESPONSE FOR UNKNOWN REASON at probe %s" % probe_id)
else:
raise Blaeu.WrongAssumption("Neither result not resultset member")
if len(result_set) == 0:
myset.sort()
set_str = " ".join(myset)
sets[set_str].total += 1
if config.display_probes:
if set_str in probes_sets:
probes_sets[set_str].append(probe_id)
else:
probes_sets[set_str] = [probe_id,]
for result_i in result_set:
try:
if "dst_addr" in result_i:
resolver = str(result_i['dst_addr'])
elif "dst_name" in result_i: # Apparently, used when there was a problem
resolver = str(result_i['dst_name'])
elif "dst_addr" in result: # Used when specifying a name server
resolver = str(result['dst_addr'])
elif "dst_name" in result: # Apparently, used when there was a problem
resolver = str(result['dst_name'])
else:
resolver = "UNKNOWN RESOLUTION ERROR"
myset = []
if "result" not in result_i:
if config.only_one_per_probe:
continue
else:
if "timeout" in result_i['error']:
myset.append("TIMEOUT")
elif "socket" in result_i['error']:
all_timeout = False
myset.append("NETWORK PROBLEM WITH RESOLVER")
else:
all_timeout = False
myset.append("NO RESPONSE FOR UNKNOWN REASON at probe %s" % probe_id)
else:
all_timeout = False
resolver_responds = True
answer = result_i['result']['abuf'] + "=="
content = base64.b64decode(answer)
msg = dns.message.from_wire(content)
if config.nsid:
for opt in msg.options:
if opt.otype == dns.edns.NSID:
myset.append("NSID: %s;" % opt.data.decode())
successes += 1
if msg.rcode() == dns.rcode.NOERROR:
probe_resolves = True
if config.answer_section:
if result_i['result']['ANCOUNT'] == 0 and config.verbose:
# If we test an authoritative server, and it returns a delegation, we won't see anything...
print("Warning: reply at probe %s has no answers: may be the server returned a delegation, or does not have data of type %s? For the first case, you may want to use --authority." % (probe_id, config.qtype), file=sys.stderr)
interesting_section = msg.answer
elif config.authority_section:
interesting_section = msg.authority
elif config.additional_section:
interesting_section = msg.additional
for rrset in interesting_section:
for rdata in rrset:
if rdata.rdtype == qtype_num:
myset.append(str(rdata)[0:MAXLEN].lower()) # We truncate because DNSKEY can be very long
if config.display_validation and (msg.flags & dns.flags.AD):
myset.append(" (Authentic Data flag) ")
if (msg.flags & dns.flags.TC):
if edns_size is not None:
myset.append(" (TRUNCATED - EDNS buffer size was %d ) " % edns_size)
else:
myset.append(" (TRUNCATED - May have to use --ednssize) ")
else:
if msg.rcode() == dns.rcode.REFUSED: # Not SERVFAIL since
# it can be legitimate (DNSSEC problem, for instance)
if config.only_one_per_probe and len(result_set) > 1: # It
# does not handle the case where there
# are several resolvers and all say
# REFUSED (probably a rare case).
if first_error == "":
first_error = "ERROR: %s" % dns.rcode.to_text(msg.rcode())
continue # Try again
else:
probe_resolves = True # NXDOMAIN or SERVFAIL are legitimate
myset.append("ERROR: %s" % dns.rcode.to_text(msg.rcode()))
myset.sort()
set_str = " ".join(myset)
sets[set_str].total += 1
if "error" not in result_i:
sets[set_str].successes += 1
if config.display_probes:
if set_str in probes_sets:
probes_sets[set_str].append(probe_id)
else:
probes_sets[set_str] = [probe_id,]
if config.display_resolvers:
if set_str in resolvers_sets:
if not (resolver in resolvers_sets[set_str]):
resolvers_sets[set_str].append(resolver)
else:
resolvers_sets[set_str] = [resolver,]
if config.display_rtt:
if "error" not in result_i:
if "result" not in result_i:
sets[set_str].rtt += result_i['rt']
else:
sets[set_str].rtt += result_i['result']['rt']
except dns.name.BadLabelType:
if not config.machine_readable:
print("Probe %s failed (bad label in name)" % probe_id, file=sys.stderr)
except dns.message.TrailingJunk:
if not config.machine_readable:
print("Probe %s failed (trailing junk)" % probe_id, file=sys.stderr)
except dns.exception.FormError:
if not config.machine_readable:
print("Probe %s failed (malformed DNS message)" % probe_id, file=sys.stderr)
if config.only_one_per_probe:
break
if not probe_resolves and first_error != "" and config.verbose:
print("Warning, probe %s has no working resolver (first error is \"%s\")" % (probe_id, first_error), file=sys.stderr)
if not resolver_responds:
if all_timeout and not config.only_one_per_probe:
if config.verbose:
print("Warning, probe %s never got reply from any resolver" % (probe_id), file=sys.stderr)
set_str = "TIMEOUT(S) on all resolvers"
sets[set_str].total += 1
else:
myset.sort()
set_str = " ".join(myset)
if config.sort:
sets_data = sorted(sets, key=lambda s: sets[s].total, reverse=True)
else:
sets_data = sets
details = []
if not config.machine_readable and config.nameserver is not None:
print("Nameserver %s" % config.nameserver)
if not config.answer_section:
if config.authority_section:
print("Authority section of the DNS responses")
elif config.additional_section:
print("Additional section of the DNS responses")
else:
print("INTERNAL PROBLEM: no section to display?")
for myset in sets_data:
detail = ""
if config.display_probes:
detail = "(probes %s)" % probes_sets[myset]
if config.display_resolvers:
detail += "(resolvers %s)" % resolvers_sets[myset]
if config.display_rtt and sets[myset].successes > 0:
detail += "Average RTT %i ms" % (sets[myset].rtt/sets[myset].successes)
if not config.machine_readable:
print("[%s] : %i occurrences %s" % (myset, sets[myset].total, detail))
else:
details.append("[%s];%i" % (myset, sets[myset].total))
if not config.machine_readable:
print(("Test #%s done at %s" % (measurement.id, time.strftime("%Y-%m-%dT%H:%M:%SZ", measurement.time))))
print("")
else:
# TODO: what if we analyzed an existing measurement?
if config.nameserver is None:
ns = "DEFAULT RESOLVER"
else:
ns = config.nameserver
print(",".join([domainname, config.qtype, str(measurement.id), "%s/%s" % (len(results), measurement.num_probes), \
time.strftime("%Y-%m-%dT%H:%M:%SZ", measurement.time), ns] + details))
././@PaxHeader 0000000 0000000 0000000 00000000026 00000000000 010213 x ustar 00 22 mtime=1663079267.0
blaeu-1.1.10/blaeu-traceroute 0000755 0001750 0001750 00000025456 14310111543 017306 0 ustar 00bortzmeyer bortzmeyer #!/usr/bin/env python3
# -*- coding: utf-8 -*-
""" Python code to start a RIPE Atlas UDM (User-Defined
Measurement). This one is for running IPv4 or IPv6 traceroute queries
to analyze routing
You'll need an API key in ~/.atlas/auth.
Stéphane Bortzmeyer
"""
import json
import time
import os
import string
import sys
import time
import socket
import pickle as pickle
import copy
import Blaeu
# If we use --format:
# import cymruwhois
config = Blaeu.Config()
# Default values
config.protocol = "UDP"
config.format = False
config.whois = True # But some networks block outgoing port 43
config.do_lookup = False
config.do_reverse_lookup = False
config.first_hop = 1
config.max_hops = 32
def is_ip_address(str):
try:
addr = socket.inet_pton(socket.AF_INET6, str)
except socket.error: # not a valid IPv6 address
try:
addr = socket.inet_pton(socket.AF_INET, str)
except socket.error: # not a valid IPv4 address either
return False
return True
def lookup_hostname(str):
try:
info = socket.getaddrinfo(str, 0, socket.AF_UNSPEC, socket.SOCK_STREAM,0, socket.AI_PASSIVE)
if len(info) > 1:
print("%s returns more then one IP address please select one" % str)
count=0
for ip in info:
count= count + 1
fa, socktype, proto, canonname, sa = ip
print("%s - %s" % (count, sa[0]))
selection=int(input("=>"))
selection = selection - 1
selected_ip=info[selection][4][0]
else:
selected_ip=info[0][4][0]
print("Using IP: %s" % selected_ip)
except socket.error:
return False
return selected_ip
def lookup_ip(ip):
try:
name, alias, addresslist = socket.gethostbyaddr(ip)
except Exception as e:
msg = "No PTR"
return msg
return name
def usage(msg=None):
print("Usage: %s target-IP-address-or-name" % sys.argv[0], file=sys.stderr)
config.usage(msg)
print("""Also:
--format or -k : downloads the results and format them in a traditional traceroute way
--simpleformat : the same, but without looking up the AS (useful if you have no whois access)
--protocol=PROTO or -j PROTO : uses this protocol (UDP, TCP or ICMP, default is %s)
--do_lookup or -d : Enables IP lookup feature (default is disabled, may become interactive if the machine has several addresses)
--do_reverse_lookup or -l : Enables reverse IP lookup feature for hops
--first_hop=N or -y N : TTL/max hop count for the first hop (default %d)
--max_hops=N or -x N : TTL/max hop count for the last hop (default %d)
""" % (config.protocol, config.first_hop, config.max_hops), file=sys.stderr)
"""For "TCP Ping"
,
you need --protocol TCP --size=0 --port=$PORT --first_hop=64
"""
def specificParse(config, option, value):
result = True
if option == "--protocol" or option == "-j":
if value.upper() != "UDP" and value.upper() != "ICMP" and value.upper() != "TCP":
usage("Protocol must be UDP or ICMP or TCP: %s rejected" % value.upper())
sys.exit(1)
config.protocol = value.upper()
elif option == "--first_hop" or option == "-y":
config.first_hop = int(value)
elif option == "--max_hops" or option == "-x":
config.max_hops = int(value)
elif option == "--format" or option == "-k":
config.format = True
elif option == "--simpleformat":
config.format = True
config.whois = False
elif option == "--do_lookup" or option == "-d":
config.do_lookup = True
elif option == "--do_reverse_lookup" or option == "-l":
config.do_reverse_lookup = True
else:
result = False
return result
args, data = config.parse("j:x:kdy:l", ["format", "simpleformat", "protocol=", "first_hop=", "max_hops=",
"do_lookup","do_reverse_lookup"], specificParse, usage)
if len(args) != 1:
usage()
sys.exit(1)
target = args[0]
if config.do_lookup:
hostname = target
target = lookup_hostname(hostname)
if not target:
print(("Unknown host name \"%s\"" % hostname), file=sys.stderr)
sys.exit(1)
else:
if not is_ip_address(target):
print("Target must be an IP address, NOT AN HOST NAME (or use --do_lookup)", file=sys.stderr)
sys.exit(1)
data["definitions"][0]["description"] = ("Traceroute %s" % target) + data["definitions"][0]["description"]
data["definitions"][0]["type"] = "traceroute"
data["definitions"][0]["protocol"] = config.protocol
data["definitions"][0]["target"] = target
if config.first_hop is not None:
data["definitions"][0]['first_hop'] = config.first_hop
if config.max_hops is not None:
data["definitions"][0]['max_hops'] = config.max_hops
if target.find(':') > -1: # TODO: or use is_ip_address(str) from blaeu-reach?
config.ipv4 = False
af = 6
if config.include is not None:
data["probes"][0]["tags"]["include"] = copy.copy(config.include)
data["probes"][0]["tags"]["include"].append("system-ipv6-works")
else:
data["probes"][0]["tags"]["include"] = ["system-ipv6-works",]
else:
config.ipv4 = True
af = 4
if config.include is not None:
data["probes"][0]["tags"]["include"] = copy.copy(config.include)
data["probes"][0]["tags"]["include"].append("system-ipv4-works")
else:
data["probes"][0]["tags"]["include"] = ["system-ipv4-works",]
data["definitions"][0]['af'] = af
if config.measurement_id is None:
if config.verbose:
print(data)
measurement = Blaeu.Measurement(data)
print("Measurement #%s %s uses %i probes" % (measurement.id,
data["definitions"][0]["description"],
measurement.num_probes))
rdata = measurement.results(wait=True, percentage_required=config.percentage_required)
print(("%s probes reported" % len(rdata)))
else:
measurement = Blaeu.Measurement(data=None, id=config.measurement_id)
rdata = measurement.results(wait=False)
if config.verbose:
print("%i results from already-done measurement #%s" % (len(rdata), measurement.id))
print(("Test #%s done at %s" % (measurement.id, time.strftime("%Y-%m-%dT%H:%M:%SZ", measurement.time))))
if config.format: # Code stolen from json2traceroute.py
if config.whois:
from cymruwhois import Client
def whoisrecord(ip):
try:
currenttime = time.time()
ts = currenttime
if ip in whois:
ASN,ts = whois[ip]
else:
ts = 0
if ((currenttime - ts) > 36000):
c = Client()
ASN = c.lookup(ip)
whois[ip] = (ASN,currenttime)
return ASN
except Exception as e:
return e
if config.whois:
try:
pkl_file = open('whois.pkl', 'rb')
whois = pickle.load(pkl_file)
except IOError:
whois = {}
# Create traceroute output
try:
for probe in rdata:
probefrom = probe["from"]
if probefrom:
if config.whois:
ASN = whoisrecord(probefrom)
if not isinstance(ASN, Exception):
asn = ASN.asn
owner = ASN.owner
else:
asn = "No AS: %s \"%s\"" % (type(ASN).__name__, ASN)
owner = "Unknown"
else:
asn = ""
owner = ""
try:
print("From: ",probefrom," ",asn," ",owner)
except Exception as e:
print("From: ", probefrom," ","AS lookup error: ",e)
print("Source address: ",probe["src_addr"])
print("Probe ID: ",probe["prb_id"])
result = probe["result"]
for proberesult in result:
ASN = {}
if "result" in proberesult:
print(proberesult["hop"]," ", end=' ')
hopresult = proberesult["result"]
rtt = []
hopfrom = ""
for hr in hopresult:
if "error" in hr:
rtt.append(hr["error"])
elif "x" in hr:
rtt.append(str(hr["x"]))
elif "edst" in hr:
rtt.append("!")
else:
try:
rtt.append(hr["rtt"])
except KeyError:
rtt.append("*")
hopfrom = hr["from"]
if config.whois:
ASN = whoisrecord(hopfrom)
if hopfrom:
try:
if config.whois:
if not isinstance(ASN, Exception):
asn = ASN.asn
owner = ASN.owner
else:
asn = "No AS: %s \"%s\"" % (type(ASN).__name__, ASN)
owner = "Unknown"
else:
asn = ""
owner = ""
if not config.do_reverse_lookup:
print(hopfrom, " ", asn, " ", owner," ",
end=' ')
else:
reverse_lookup = lookup_ip(hopfrom)
print(hopfrom, " ", reverse_lookup, " ", asn, " ",
owner, " ", end=' ')
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
print(hopfrom, " Lookup failed because of", exc_type.__name__, "(",
exc_value, ") ", end=' ')
print(rtt)
else:
print("Error: ", proberesult["error"])
print("")
finally:
if config.whois:
pkl_file = open('whois.pkl', 'wb')
pickle.dump(whois, pkl_file)
././@PaxHeader 0000000 0000000 0000000 00000000033 00000000000 010211 x ustar 00 27 mtime=1706190263.134136
blaeu-1.1.10/blaeu.egg-info/ 0000755 0001750 0001750 00000000000 14554462667 016715 5 ustar 00bortzmeyer bortzmeyer ././@PaxHeader 0000000 0000000 0000000 00000000026 00000000000 010213 x ustar 00 22 mtime=1706190262.0
blaeu-1.1.10/blaeu.egg-info/PKG-INFO 0000644 0001750 0001750 00000005767 14554462666 020030 0 ustar 00bortzmeyer bortzmeyer Metadata-Version: 2.1
Name: blaeu
Version: 1.1.10
Summary: Tools to create (and analyze) RIPE Atlas network measurements
Home-page: https://framagit.org/bortzmeyer/blaeu
Author: Stéphane Bortzmeyer
Author-email: stephane+frama@bortzmeyer.org
License: BSD
Keywords: networking atlas monitoring ip ping traceroute dig
Classifier: Development Status :: 4 - Beta
Classifier: Intended Audience :: System Administrators
Classifier: Intended Audience :: Telecommunications Industry
Classifier: Topic :: System :: Networking
Classifier: License :: OSI Approved :: MIT License
Classifier: Programming Language :: Python :: 3
Requires-Python: >=3
Provides-Extra: dev
License-File: LICENCE
Blaeu, creating measurements on RIPE Atlas probes
=================================================
This is a set of `Python `__ programs to start
distributed Internet measurements on the network of `RIPE Atlas
probes `__, and to analyze their results.
For installation, you can use usual Python tools, for instance:
::
pip3 install blaeu
(On a Debian machine, the prerequitises are packages python3-pip,
python3-openssl, python3-dnspython, and python3-cymruwhois. This is only
if you install manually, otherwise pip3 will install the dependencies.)
Usage requires a RIPE Atlas API key (which itself requires a RIPE
account), and RIPE Atlas credits. If you don’t have a RIPE account,
`register first `__. Once you have an account,
`create a key `__, grant it the right to
``schedule a new measurement``, and put the key in ``~/.atlas/auth``. If
you don’t have Atlas credits, host a probe,or become a
`LIR `__ or ask a
friend.
You can then use the four programs (``-h`` will give you a complete list
of their options):
- ``blaeu-reach target-IP-address`` (test reachability of the target,
like ``ping``)
- ``blaeu-traceroute target-IP-address`` (like ``traceroute``)
- ``blaeu-resolve name`` (use the DNS to resolve the name)
- ``blaeu-cert name`` (display the PKIX certificate)
You may also be interested by `my article at RIPE
Labs `__.
Blaeu requires Python 3.
Note that `the old
version `__
ran on Python 2 but is no longer maintained. (It was `partially
documented at RIPE
Labs `__.)
Name
----
It comes from the `famous Dutch
cartographer `__. The logo
of the project comes from his “Theatrum Orbis Terrarum” (see `the
source `__).
Reference site
--------------
`On FramaGit `__
Author
------
Stéphane Bortzmeyer stephane+frama@bortzmeyer.org
././@PaxHeader 0000000 0000000 0000000 00000000026 00000000000 010213 x ustar 00 22 mtime=1706190263.0
blaeu-1.1.10/blaeu.egg-info/SOURCES.txt 0000644 0001750 0001750 00000000366 14554462667 020606 0 ustar 00bortzmeyer bortzmeyer Blaeu.py
LICENCE
MANIFEST.in
README.rst
blaeu-cert
blaeu-reach
blaeu-resolve
blaeu-traceroute
setup.py
blaeu.egg-info/PKG-INFO
blaeu.egg-info/SOURCES.txt
blaeu.egg-info/dependency_links.txt
blaeu.egg-info/requires.txt
blaeu.egg-info/top_level.txt ././@PaxHeader 0000000 0000000 0000000 00000000026 00000000000 010213 x ustar 00 22 mtime=1706190262.0
blaeu-1.1.10/blaeu.egg-info/dependency_links.txt 0000644 0001750 0001750 00000000001 14554462666 022762 0 ustar 00bortzmeyer bortzmeyer
././@PaxHeader 0000000 0000000 0000000 00000000026 00000000000 010213 x ustar 00 22 mtime=1706190262.0
blaeu-1.1.10/blaeu.egg-info/requires.txt 0000644 0001750 0001750 00000000057 14554462666 021316 0 ustar 00bortzmeyer bortzmeyer cymruwhois
dnspython
pyopenssl
[dev]
pypandoc
././@PaxHeader 0000000 0000000 0000000 00000000026 00000000000 010213 x ustar 00 22 mtime=1706190262.0
blaeu-1.1.10/blaeu.egg-info/top_level.txt 0000644 0001750 0001750 00000000006 14554462666 021442 0 ustar 00bortzmeyer bortzmeyer Blaeu
././@PaxHeader 0000000 0000000 0000000 00000000033 00000000000 010211 x ustar 00 27 mtime=1706190263.134136
blaeu-1.1.10/setup.cfg 0000644 0001750 0001750 00000000046 14554462667 015754 0 ustar 00bortzmeyer bortzmeyer [egg_info]
tag_build =
tag_date = 0
././@PaxHeader 0000000 0000000 0000000 00000000026 00000000000 010213 x ustar 00 22 mtime=1706173843.0
blaeu-1.1.10/setup.py 0000644 0001750 0001750 00000012512 14554422623 015632 0 ustar 00bortzmeyer bortzmeyer # -*- coding: utf-8 -*-
"""See:
https://packaging.python.org/en/latest/distributing.html
https://github.com/pypa/sampleproject
"""
# Always prefer setuptools over distutils
from setuptools import setup, find_packages
# To use a consistent encoding
from codecs import open
from os import path
here = path.abspath(path.dirname(__file__))
# Get the long description from the README file
if not path.exists('README.rst'): # TODO: and test it is newer than README.md
import pypandoc
rst = pypandoc.convert('README.md', 'rst')
f = open('README.rst','w+')
f.write(rst)
f.close()
with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
long_description = f.read()
# Arguments marked as "Required" below must be included for upload to PyPI.
# Fields marked as "Optional" may be commented out.
setup(
name='blaeu', # Required
# Versions should comply with PEP 440:
# https://www.python.org/dev/peps/pep-0440/
version="1.1.10", # WARNING: if you modify it here, also change Blaeu.py https://packaging.python.org/guides/single-sourcing-package-version/#single-sourcing-the-version
# This is a one-line description or tagline of what your project does. This
# corresponds to the "Summary" metadata field:
# https://packaging.python.org/specifications/core-metadata/#summary
description='Tools to create (and analyze) RIPE Atlas network measurements', # Required
# This field corresponds to the "Description" metadata field:
# https://packaging.python.org/specifications/core-metadata/#description-optional
long_description=long_description, # Optional
# This should be a valid link to your project's main homepage.
#
# This field corresponds to the "Home-Page" metadata field:
# https://packaging.python.org/specifications/core-metadata/#home-page-optional
url='https://framagit.org/bortzmeyer/blaeu', # Optional
# This should be your name or the name of the organization which owns the
# project.
author='Stéphane Bortzmeyer', # Optional
# This should be a valid email address corresponding to the author listed
# above.
author_email='stephane+frama@bortzmeyer.org', # Optional
license = 'BSD',
# Classifiers help users find your project by categorizing it.
#
# For a list of valid classifiers, see
# https://pypi.python.org/pypi?%3Aaction=list_classifiers
classifiers=[ # Optional
# How mature is this project? Common values are
# 3 - Alpha
# 4 - Beta
# 5 - Production/Stable
'Development Status :: 4 - Beta',
# Indicate who your project is intended for
'Intended Audience :: System Administrators',
'Intended Audience :: Telecommunications Industry',
'Topic :: System :: Networking',
# Pick your license as you wish
'License :: OSI Approved :: MIT License',
# Specify the Python versions you support here. In particular, ensure
# that you indicate whether you support Python 2, Python 3 or both.
'Programming Language :: Python :: 3'
],
# This field adds keywords for your project which will appear on the
# project page. What does your project relate to?
#
# Note that this is a string of words separated by whitespace, not a list.
keywords='networking atlas monitoring ip ping traceroute dig', # Optional
# You can just specify package directories manually here if your project is
# simple. Or you can use find_packages().
#
# Alternatively, if you just want to distribute a single Python file, use
# the `py_modules` argument instead as follows, which will expect a file
# called `my_module.py` to exist:
#
py_modules=["Blaeu"],
#
packages=[], # Required
# This field lists other packages that your project depends on to run.
# Any package you put here will be installed by pip when your project is
# installed, so they must be valid existing projects.
#
# For an analysis of "install_requires" vs pip's requirements files see:
# https://packaging.python.org/en/latest/requirements.html
install_requires=['cymruwhois', 'pyopenssl', 'dnspython'], # Optional
# List additional groups of dependencies here (e.g. development
# dependencies). Users will be able to install these using the "extras"
# syntax, for example:
#
# $ pip install sampleproject[dev]
#
# Similar to `install_requires` above, these must be valid existing
# projects.
extras_require={
'dev': ['pypandoc']
# Optional
},
python_requires='>=3',
# If there are data files included in your packages that need to be
# installed, specify them here.
#
# If using Python 2.6 or earlier, then these have to be included in
# MANIFEST.in as well.
package_data={ # Optional
},
# Although 'package_data' is the preferred approach, in some case you may
# need to place data files outside of your packages. See:
# https://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files
#
# In this case, 'data_file' will be installed into '/my_data'
data_files=[], # Optional
# To provide executable scripts, entry points are officially
# recommended but way too hard for me.
scripts=['blaeu-reach', 'blaeu-resolve', 'blaeu-traceroute', 'blaeu-cert'],
)