# HG changeset patch # User Jeff Hammel # Date 1486662424 28800 # Node ID 26e919a36f86ea9da305b7cce6219eb74f11b75a speedtest containerized dispatching software diff -r 000000000000 -r 26e919a36f86 README.md --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/README.md Thu Feb 09 09:47:04 2017 -0800 @@ -0,0 +1,200 @@ +# Speedtest + +## The Pattern + +`speedtest` +will use this pattern for outer loop testing: + +- a [Docker](http://docker.io/) container is created from a + `speedtest` script, + and a docker + [CMD] + (https://docs.docker.com/v1.8/reference/builder/#cmd) + that will run + the appropriate test and post a JSON blob to an + [Elasticsearch](https://www.elastic.co/) node + +- the Docker container is dispatched to a container service + such that it may be run at + cloud scale and from a variety of regions + +- the test [CMD] + (http://docs.docker.com/engine/reference/builder/#cmd) + of the container contains steps that will generate + JSON data and post this to an Elasticsearch node. This + Elasticsearch data may then be analyzed and utilized for reporting + of system status. + +This pattern is of general utility for blackbox testing and monitoring +of a web system under test. Monitoring may be performed in a continuous +and ongoing basis, the Elasticsearch data combed for performance information. + + +## Installation + +`speedtest` follows fairly standard +[python packaging] +(https://python-packaging-user-guide.readthedocs.org/en/latest/) +practices. We do need other software, like +[docker](http://docker.io), to help us along. + +Let's get started! + + +## What Speedtest Gives You + +`speedtest` attempts to provide a suite of tools for making worldwide +dockerized speedtests a reality. Many of these are provide in terms +of [setuptools](https://pythonhosted.org/setuptools/) +[entry points] +(https://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins), +including +[console scripts] +(https://pythonhosted.org/setuptools/setuptools.html#automatic-script-creation). + +Several console scripts are also installed with `speedtest`. +All provided console scripts utilize +[argparse](https://docs.python.org/2/library/argparse.html) +for their command line interface. Detailed usage information for a +script may be obtained by using the `--help` command line switch. +For example: `speedtest --help` . + + +## About Speedtest + +`speedtest` is a collection of scripts and utilities to make real the +pattern of docker container dispatch for data +gathering via JSON output to Elasticsearch. + +While `Dockerfile`s are a good abstraction layer for building +[linux containers] +(https://en.wikipedia.org/wiki/LXC), +often the run steps must be automated and parameterized. + +## speedtest with local results + +Usage information is available with the `--help` flag: + + speedtest --help + +Arbitrary metadata may be added with the `--variables` flag. While the results processing and information available should be refined, right now we are erring on the side of too much, which is at least fairly flexible. + +To run a speedtest versus the URL http://k0s.org/playground/1M.bin adding the variable `foo` with value of `bar` with local JSON results measuring every 100k, do: + + speedtest --variables foo=bar --format json --chunk $((100*1024)) http://k0s.org/playground/1M.bin + + +## Containers + +### Containerized Download Speed Testing + +`speedtest.py` + +(and, equivalently, the `speedtest` [console script] +(http://python-packaging.readthedocs.org/en/latest/command-line-scripts.html#the-console-scripts-entry-point)) +measures download speed over time given a URL: + + # speedtest http://k0s.org/playground/1M.bin + ... + {"duration": 1.7867929935455322, "cumulative": 586848, "speed": 209367, "size": 1048576} + + +### Containerized Dispatching + +- pull the Docker image from a container registry +- run the Docker image + +### Container Registry + +In order to make use of containerized dispatch, you may want a +[docker registry](https://docs.docker.com/registry/): + +- https://docs.docker.com/registry/deploying/ +- https://www.docker.com/docker-trusted-registry +- https://github.com/docker/distribution +- https://www.docker.com/aws + +[Docker hub](https://hub.docker.com/) +has an official [Docker](https://www.docker.com) +repository for a +[Docker registry] +(https://hub.docker.com/_/registry/). How meta is that? You can get +started just by running: + + docker run registry:2 + + +## Elasticsearch + +_"Elasticsearch is a search server based on Lucene. It provides a +distributed, multitenant-capable full-text search engine with a HTTP +web interface and schema-free JSON documents. Elasticsearch is +developed in Java and is released as open source under the terms of +the Apache License."_, +[Wikipedia](https://en.wikipedia.org/wiki/Elasticsearch) + +The schema-free data store together with the easy RESTful interface +make [Elasticsearch] +(https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html) +a great place for `speedtest` to put its results. + +There is a docker container for Elasticsearch available: +https://hub.docker.com/_/elasticsearch/ + +The results of `speedtest` may be posted to Elasticsearch: + + # curl -XPOST localhost:9200/speedtest/results?pretty -d "$(speedtest http://k0s.org/playground/1M.bin | tail -n 1)" + { + "_index" : "speedtest", + "_type" : "results", + "_id" : "AVGNmWMTVkaacKUEikLl", + "_version" : 1, + "_shards" : { + "total" : 2, + "successful" : 1, + "failed" : 0 + }, + "created" : true + } + +However, for lots of data we probably want to use the +[bulk API] +(https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html) + + # speedtest 'http://k0s.org/playground/1M.bin' --fmt elasticsearch --chunk $((1024*1024)) -o 'http://1.2.3.4:32793/speedtest/results/_bulk' + {"took":1,"errors":false,"items":[{"create":{"_index":"speedtest","_type":"results","_id":"AVVU7PIeZtDhCquHQrZV","_version":1,"status":201}}]} + +Note that the index and type *must* be specified in the output URL for this to work. +See https://www.elastic.co/blog/index-vs-type for clarification of why this is a +good thing. + + +JSON blobs are uploaded to an Elasticsearch instance. + + +## Links + + , , + |`-._ _.-'| + \ `-' / + `-._ _.-' + `#' + ~~~~~~~~~~~~~ + +[Docker](http://docker.com/): + +- https://docs.docker.com/machine/get-started/ +- https://github.com/victorlin/crane +- https://docker-py.readthedocs.org/en/latest/ + + +Elasticsearch: +- [The Definitive Guide](https://www.elastic.co/guide/en/elasticsearch/guide/index.html) +- https://www.elastic.co/guide/en/elasticsearch/reference/current/_modifying_your_data.html +- https://hub.docker.com/_/elasticsearch/ +- [Python Elasticsearch Client] + (http://elasticsearch-py.readthedocs.org/en/master/index.html) +- https://bitquabit.com/post/having-fun-python-and-elasticsearch-part-1/ + + +[Jeff Hammel](https://hub.docker.com/u/jehammel/) diff -r 000000000000 -r 26e919a36f86 TODO.md --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/TODO.md Thu Feb 09 09:47:04 2017 -0800 @@ -0,0 +1,13 @@ +- Genericize Docker templatization: + A generic Docker project template should be + made and templates should be made to extend that. + +- Much of what is done, or what is desired to be done, as far as + interacting with [Docker](http://docker.io/) my have a good library + solution. For a proof of concept, I didn't want to stray to much to + explore versus getting what I wanted working. But, in general, for + next steps third-party libraries should be vetted and standardized + on. A potential winner: + https://docker-py.readthedocs.org/en/latest/api/#create_container + + Client(os.environ['DOCKER_HOST']).create_container('elasticsearch:1.5') diff -r 000000000000 -r 26e919a36f86 example/bulk.json --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/example/bulk.json Thu Feb 09 09:47:04 2017 -0800 @@ -0,0 +1,6 @@ +{"index": {}} +{"url": "http://k0s.org/playground/1M.bin", "duration": 0.5988240242004395, "cumulative": 1749348, "speed": 31122951, "size": 1047552} +{"index": {}} +{"url": "http://k0s.org/playground/1M.bin", "duration": 0.6530699729919434, "cumulative": 1600906, "speed": 3821145, "size": 1045504} +{"index": {}} +{"url": "http://k0s.org/playground/1M.bin", "duration": 0.6531820297241211, "cumulative": 1602199, "speed": 9138228, "size": 1046528} \ No newline at end of file diff -r 000000000000 -r 26e919a36f86 setup.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/setup.py Thu Feb 09 09:47:04 2017 -0800 @@ -0,0 +1,51 @@ +import os +import sys + +version = "0.0" +dependencies = ['MakeItSo', + 'requests', + 'elasticsearch>=1.0.0,<2.0.0' # Elasticsearch 1.x +] + +if sys.version_info < (2, 7): + # `argparse` is incorporated into cython's stdlib in python 2.7 : + # https://docs.python.org/2/library/argparse.html + dependencies.append('argparse') + +# allow use of setuptools/distribute or distutils +kw = {} +try: + from setuptools import setup + kw['entry_points'] = """ + [console_scripts] + docker-images = speedtest.docker_images:main + docker-run = speedtest.docker_run:main + elastic = speedtest.elasticsearch:main + py-eval = speedtest.eval:main + speedtest = speedtest.speedtest:main +""" + kw['install_requires'] = dependencies +except ImportError: + from distutils.core import setup + kw['requires'] = dependencies + +try: + here = os.path.dirname(os.path.abspath(__file__)) + description = file(os.path.join(here, 'README.md')).read() +except IOError: + description = '' + + +setup(name='speedtest', + version=version, + description="", + long_description=description, + classifiers=[], # Get strings from http://www.python.org/pypi?%3Aaction=list_classifiers + author='Jeff Hammel', + author_email='k0scist@gmail.com', + license='', + packages=['speedtest'], + include_package_data=True, + zip_safe=False, + **kw + ) diff -r 000000000000 -r 26e919a36f86 speedtest/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/speedtest/__init__.py Thu Feb 09 09:47:04 2017 -0800 @@ -0,0 +1,2 @@ +# + diff -r 000000000000 -r 26e919a36f86 speedtest/docker.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/speedtest/docker.py Thu Feb 09 09:47:04 2017 -0800 @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- + +""" +docker interface using the command line +""" + +# See also: +# - https://docker-py.readthedocs.org/en/latest/ +# - https://github.com/victorlin/crane + +# imports +import argparse +import json +import os +import subprocess +import sys +import time +import uuid +from parse import parse_columns + +# docker CLI conventions +NONE = '' + +def build(directory): + """ + # docker build . + Sending build context to Docker daemon 2.048 kB + Step 1 : FROM centos:6 + ---> 3bbbf0aca359 + Successfully built 3bbbf0aca359 + """ + + # ensure directory exists + if not os.path.isdir(directory): + raise AssertionError("Not a directory: {directory}".format(directory=directory)) + + # build it! + output = subprocess.check_output(['docker', 'build', '.'], + cwd=directory) + + # return the image ID (e.g. `3bbbf0aca359`) + # for further interaction + # See also possibly + # https://docker-py.readthedocs.org/en/latest/ + lines = output.strip().splitlines() + final = lines[-1] + assert final.startswith('Successfully built') + return final.rsplit()[-1] + + +def images(): + """list images locally""" + + headers = ['REPOSITORY', + 'TAG', + 'IMAGE ID', + 'CREATED', + 'VIRTUAL SIZE'] + output = subprocess.check_output(['docker', 'images']) + images = parse_columns(output, headers) + for image in images: + for key in ('REPOSITORY', 'TAG'): + if image[key] == NONE: + image[key] = None + return images + + +def ps(): + """return `docker ps` information""" + + headers = ['CONTAINER ID', + 'IMAGE' + 'COMMAND', + 'CREATED', + 'STATUS', + 'PORTS', + 'NAMES'] + output = subprocess.check_output(['docker', 'ps']) + return parse_columns(output, headers) + + +def port(container): + """ + returns the port mapping of a container + + (Ebeneezer)# docker port 9b74817b0f30 + 9200/tcp -> 0.0.0.0:32769 + 9300/tcp -> 0.0.0.0:32768 + """ + + output = subprocess.check_output(['docker', 'port', container]) + output = output.strip() + port_mapping = {} + for line in output.splitlines(): + guest_port, host_port = [i.strip() + for i in line.split('->')] + guest_port, protocol = guest_port.split('/') + guest_port = int(guest_port) + address, host_port = host_port.split(':') + host_port = int(host_port) + port_mapping[guest_port] = host_port + return port_mapping + + +def inspect(container): + """ + Return low-level information on a container or image: + + https://docs.docker.com/engine/reference/commandline/inspect/ + """ + + return json.loads(subprocess.check_output(['docker', 'inspect', container])) + +def remove_image(*image): + """remove a docker image""" + + subprocess.check_output(["docker", "rmi"] + list(image)) + + +class Container(object): + """context manager for containers""" + + def __init__(self, image, name=None, timeout=120.): + if name is None: + name = uuid.uuid1() + self.name = str(name) + self.process = subprocess.Popen(['docker', 'run', '-P', '--name', self.name, image]) + self.timeout = timeout + + # TODO: + + # * --cidfile: + # https://docs.docker.com/engine/reference/commandline/run/ + + # * poll `self.running` until we see if the container is + # running or the process has exited + + def running(self): + """returns if the container is running""" + + def __enter__(self): + return self + + def kill(self): + """kill the running container""" + self.process.kill() + + def __exit__(self, type, value, traceback): + self.kill() diff -r 000000000000 -r 26e919a36f86 speedtest/docker_images.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/speedtest/docker_images.py Thu Feb 09 09:47:04 2017 -0800 @@ -0,0 +1,54 @@ +#!/usr/bin/env python + +""" +docker image manipulation +""" + +# imports +import argparse +import json +import sys +import docker + +def main(args=sys.argv[1:]): + """CLI""" + + # parse command line + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--repo', '--respoistory', dest='repository', + nargs='?', default=False, const=None, + help="select images of this repository ( by default)") + parser.add_argument('--count', dest='count', + action='store_true', default=False, + help="print count of images and exit") + parser.add_argument('--rm', '--remove', dest='remove', + action='store_true', default=False, + help="remove selected images") + options = parser.parse_args(args) + + # get images + images = docker.images() + + if not options.repository == False: + # select images by repository + images = [image for image in images + if image['REPOSITORY'] == options.repository] + + if options.remove: + if options.repository == False: + parser.error("Cowardly refusing to remove all images") + images = [image['IMAGE ID'] for image in images] + docker.remove_image(*images) + + # refresh what we output: + # just use all images for now + images = docker.images() + + # output + if options.count: + print len(images) + else: + print (json.dumps(images, indent=2)) + +if __name__ == "__main__": + main() diff -r 000000000000 -r 26e919a36f86 speedtest/docker_run.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/speedtest/docker_run.py Thu Feb 09 09:47:04 2017 -0800 @@ -0,0 +1,50 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +build and run a docker container +""" + + +# imports +import argparse +import docker +import os +import subprocess +import sys + + +def main(args=sys.argv[1:]): + """CLI""" + + # parse command line + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('dockerfile', + help='docker file or directory') + parser.add_argument('-i', '--it', dest='interactive', + action='store_true', default=False, + help="run the container interactively") + options = parser.parse_args(args) + + # sanity + if not os.path.exists(options.dockerfile): + parser.error("Path '{0}' does not exist".format(options.dockerfile)) + options.dockerfile = os.path.abspath(options.dockerfile) + if os.path.isfile(options.dockerfile): + options.dockerfile = os.path.dirname(options.dockerfile) + assert os.path.isdir(options.dockerfile) + + # build the container + container = docker.build(options.dockerfile) + + # run it! + command = ['docker', 'run'] + if options.interactive: + command.append('-it') + command.append(container) + print "Running:" + print subprocess.list2cmdline(command) + subprocess.call(command) + +if __name__ == '__main__': + main() diff -r 000000000000 -r 26e919a36f86 speedtest/eval.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/speedtest/eval.py Thu Feb 09 09:47:04 2017 -0800 @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +evaluate a bash script or command +""" + +# imports +import argparse +import os +import subprocess +import sys +import tempfile + + +def parse_env(output): + """parse environment content a la `env` output""" + + output = output.strip() + lines = output.splitlines() + return dict([line.split('=', 1) for line in lines]) + + +def source(src, update=True): + """ + eval a file and return the environment dictionary + """ + command = 'set -e && source "{src}" && env'.format(src=src) + output = subprocess.check_output(command, shell=True) + env = parse_env(output) + if update: + os.environ.update(env) + return env + + +def eval_command(command, update=True): + """ + evaluate a command + + command -- command to evaluate + update -- whether to update os.environ with result + """ + + output = subprocess.check_output(command, shell=True) + fd, name = tempfile.mkstemp() + try: + os.write(fd, output) + os.close(fd) + env = source(name, update=update) + finally: + try: + os.remove(name) + except: + pass + return env + + +def main(args=sys.argv[1:]): + + # parse command line + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('command', help="command to run") + options = parser.parse_args(args) + + # keep copy of the old env + old_env = os.environ.copy() + + # calcaulate the new environment + new_env = eval_command(options.command) + + # compute difference + updated = [] + for key, value in new_env.items(): + if value != old_env.get(key): + updated.append(key) + updated.sort() + + # output + for key in updated: + print ('{key}={value}'.format(key=key, value=new_env[key])) + + +if __name__ == '__main__': + main() diff -r 000000000000 -r 26e919a36f86 speedtest/parse.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/speedtest/parse.py Thu Feb 09 09:47:04 2017 -0800 @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +""" +parsing functions for speedtest +""" + +def parse_columns(output, headers=None): + """ + parse column delimited output: + + >>> parse_columns(''' + NAME ACTIVE DRIVER STATE URL SWARM + default - virtualbox Stopped + ''') + [{'NAME': 'default', 'ACTIVE': '-', 'DRIVER': 'virtualbox', 'STATE': 'Stopped': 'URL': '', 'SWARM': ''}] + """ + lines = output.strip().splitlines() + if not lines: + return [] + header_line = lines.pop(0) + if not headers: + headers = header_line.strip().split() + start = 0 + indices = [] + for header in headers: + indices.append(header_line.find(header, start)) + start = indices[-1] + len(header) + indices = zip(indices, indices[1:]+[None]) + rows = [[line[start:end].strip() for start, end, in indices] + for line in lines] + rows = [dict(zip(headers, row)) + for row in rows] + return rows diff -r 000000000000 -r 26e919a36f86 speedtest/speedtest.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/speedtest/speedtest.py Thu Feb 09 09:47:04 2017 -0800 @@ -0,0 +1,225 @@ +#!/usr/bin/env python + +""" +measure download speed of a URL +""" + +# imports +import argparse +import csv +import json +import os +import requests +import sys +import time +from StringIO import StringIO + +string = (str, unicode) + +### library functions + +def is_url(path): + return '://' in path + + +### Results handling + +class SpeedtestResults(object): + """ABC for speedtest results""" + + columns = ['duration', # seconds + 'size', # bytes + 'speed', # bytes/s + 'cumulative' # bytes/s + ] + # TODO: + # speed and cumulative are derived quantities; + # this ABC should calculate them + + def __init__(self, url, method='POST'): + self.url = url + self.rows = [] + self.method = method + + def add(self, duration, size, speed, cumulative): + """add a particular measurement""" + + raise NotImplementedError("Abstract Base Class") + + def body(self): + """results body""" + # TODO: e.g. post-processing + raise NotImplementedError("Abstract Base Class") + + def write(self, output): + + # get body + body = self.body() + + # write it! + if isinstance(output, str): + if is_url(output): + self.write_http(output, body) + else: + with open(output, 'w') as f: + f.write(body) + else: + output.write(body) + output.flush() + + def write_http(self, url, data): + """'write' to HTTP""" + + response = requests.request(self.method, url, data=data) + print (response.text) + response.raise_for_status() + + +class CSVResults(SpeedtestResults): + """output CSV speedtest results""" + + def add(self, duration, size, speed, cumulative): + self.rows.append([locals()[column] for column in self.columns]) + + def body(self): + buffer = StringIO() + writer = csv.writer(buffer) + writer.writerows(self.rows) + return buffer.getvalue() + + +class JSONResults(SpeedtestResults): + """output JSON speedtest results""" + + def __init__(self, url, data=None): + SpeedtestResults.__init__(self, url) + self.base_data = {'url': self.url} + self.base_data.update(data or {}) + + def add(self, duration, size, speed, cumulative): + data = self.base_data.copy() + data.update(dict([(column, locals()[column]) for column in self.columns])) + self.rows.append(data) + + def body(self): + return '\n'.join([json.dumps(row) for row in self.rows]) + + +class ElasticsearchResults(JSONResults): + """output Elasticsearch results""" + + def body(self): + # see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html + # this assumes that the `index` and `type` are specified by the URL + # while less than wonderful, I think that is fair + master = {'index': {}} + return '\n'.join([json.dumps(master) + '\n' + json.dumps(row) + for row in self.rows]) + '\n' + + +results_handlers = {'csv': CSVResults, + 'json': JSONResults, + 'elasticsearch': ElasticsearchResults} + + +### CLI + +class SpeedtestParser(argparse.ArgumentParser): + """parser for speedtest""" + + def __init__(self, **kwargs): + kwargs.setdefault('description', __doc__) + kwargs.setdefault('formatter_class', + argparse.ArgumentDefaultsHelpFormatter) + argparse.ArgumentParser.__init__(self, **kwargs) + self.add_arguments() + self.options = None + + def keyvalue(self, string): + """argparse key-value pair""" + if '=' not in string: + self.error("Expected '=' in {string}".format(string=string)) + return string.split('=', 1) + + def add_arguments(self): + self.add_argument('url') + self.add_argument('-o', '--output', dest='output', + help="output file or URL, or stdout by default") + self.add_argument('--variables', dest='variables', + nargs='+', type=self.keyvalue, default=(), + help="additional key-value pairs to add to each record") + self.add_argument('--fmt', '--format', dest='format', + choices=results_handlers.keys(), default='json', + help="results handler") + self.add_argument('--chunk', '--chunk-size', dest='chunk_size', + type=int, default=1024, + help="chunk size in bytes for streaming") + + def parse_args(self, *args, **kw): + options = argparse.ArgumentParser.parse_args(self, *args, **kw) + self.validate(options) + self.options = options + return self.options + + def validate(self, options): + """validate options""" + + def results_handler(self): + """return appropriate results handler according to arguments""" + + # ensure that we've called `parse_args` to parse the arguments + assert self.options is not None + if self.options is None: + raise AssertionError("`results_handler` must be called after arguments are parse") + + # get kwargs for the chosen format + format = self.options.format + kwargs = {} + if format in ('json', 'elasticsearch'): + data = dict(self.options.variables) + # only do hostname for now + hostname = os.environ.get('HOSTNAME') + if hostname: + data['hostname'] = hostname + kwargs['data'] = data + # instantiate and return handler + return results_handlers[format](self.options.url, **kwargs) + + +def main(args=sys.argv[1:]): + """CLI""" + + # parse CLI + parser = SpeedtestParser() + options = parser.parse_args(args) + + # setup output + output = parser.results_handler() + + # start the request + start = time.time() + response = requests.get(options.url, stream=True, timeout=30) + + # iterate through the response + size = 0 + last = start + for chunk in response.iter_content(options.chunk_size): + now = time.time() + duration = now - start + size += len(chunk) + speed = len(chunk)/(now - last) + cumulative = size/duration + last = now + + # add results to output + output.add(duration=duration, + size=size, + speed=int(speed), + cumulative=int(cumulative)) + + # output + output.write(options.output or sys.stdout) + + +if __name__ == '__main__': + main() diff -r 000000000000 -r 26e919a36f86 speedtest/template.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/speedtest/template.py Thu Feb 09 09:47:04 2017 -0800 @@ -0,0 +1,95 @@ +#!/usr/bin/env python + +""" +Dockerfile templatization and boilerplate for speedtest +""" + + +# imports +import argparse +import os +import shutil +import sys +import tempfile +from .docker import build +from makeitso.template import MakeItSoTemplate +from makeitso.template import Variable + + +# module globals +here = os.path.dirname(os.path.abspath(__file__)) + + +class SpeedtestTemplate(MakeItSoTemplate): + """ + template for dockerized speedtests + """ + + templates = [('templates', 'speedtest')] + modules = ['speedtest.py'] + vars = [Variable('url', 'URL to run speedtest against'), + Variable('output', 'where to output speedtest results to')] + look = False + + def pre(self, variables, output): + """before the template is rendered""" + + # set script variables + variables['scripts'] = [script + for script in self.modules] + + def post(self, variables, output): + """after the template is rendered""" + + # copy script to context directory + for script in self.modules: + script = os.path.join(here, script) + shutil.copy2(script, output) + + +def main(args=sys.argv[1:]): + """CLI""" + + # parse command line + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument('url', + help="URL to speedtest") + parser.add_argument('-o', '--output', dest='output', + help="output directory") + parser.add_argument('--build', dest='build', + action='store_true', default=False, + help="build the container") + options = parser.parse_args(args) + + # ensure output directory + working_dir = options.output + if working_dir is None: + if not options.build: + parser.error("Either `--build`, `--output`, or both must be specified") + working_dir = tempfile.mkdtemp() + else: + if os.path.exists(working_dir): + if not os.path.isdir(working_dir): + raise AssertionError("Not a directory: {}".format(working_dir)) + else: + os.makedirs(working_dir) + + # render the template + variables = {'url': options.url, + 'output': '/dev/stdout' + } + template = SpeedtestTemplate() + template.substitute(variables, working_dir) + + if options.build: + # build the container + build(working_dir) + + # cleanup + if not options.output: + shutil.rmtree(working_dir, ignore_errors=True) + + +if __name__ == '__main__': + main() diff -r 000000000000 -r 26e919a36f86 speedtest/templates/speedtest/Dockerfile --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/speedtest/templates/speedtest/Dockerfile Thu Feb 09 09:47:04 2017 -0800 @@ -0,0 +1,37 @@ +# image +FROM centos:6 + +# environment +ENV working_dir /opt/env +WORKDIR $working_dir +RUN mkdir -p $working_dir + +# install dependencies: +# python-argparse python-requests +RUN yum clean all +RUN yum -y update +RUN yum -y install python-argparse python-requests + +# install EPEL: +# Alternative RUN rpm -ivh http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm +RUN yum -y install epel-release + +# install pip: +# https://pypi.python.org/pypi/pip +RUN yum -y install python-pip +RUN pip install --upgrade pip + +# install boto: +RUN pip install boto + +# COPY scripts: +# http://docs.docker.com/engine/reference/builder/#copy +{{for script in scripts}} +COPY {{script}} ${working_dir} +{{endfor}} + +# setup the command for running later +CMD ["python", \ + "speedtest.py", \ + "{{url}}", \ + "-o", "{{output}}"] diff -r 000000000000 -r 26e919a36f86 tests/data/docker_images.out --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/data/docker_images.out Thu Feb 09 09:47:04 2017 -0800 @@ -0,0 +1,67 @@ +REPOSITORY TAG IMAGE ID CREATED VIRTUAL SIZE + d88ae212a54d 8 days ago 354.4 MB + aeb8220dd452 8 days ago 354.4 MB +ubuntu latest 8444cb1ed763 2 weeks ago 122 MB +elasticsearch latest 73a0d099b9fb 2 weeks ago 345 MB + f97d6afa84ee 2 weeks ago 775.7 MB +gcr.io/google_containers/hyperkube-amd64 v1.2.4 b65f775dbf89 4 weeks ago 316.8 MB +docker dind 5b17bfdd0506 4 weeks ago 80.41 MB + 37f785c8794f 9 weeks ago 344.9 MB + b22cb196da83 10 weeks ago 775.7 MB + 9a812c3ae65d 3 months ago 453.4 MB + 3e4251a30f93 3 months ago 775.7 MB + 7c8a5cff77a8 3 months ago 775.7 MB + 2cde8a91395c 3 months ago 775.7 MB + 866510d638cd 3 months ago 733.6 MB + 5dfb583b8634 3 months ago 733.5 MB + 1116005cf94b 3 months ago 616.5 MB +centos latest 2933d50b9f77 3 months ago 196.6 MB + 7d1f55980b76 5 months ago 190.6 MB + eb2dea6c4a03 5 months ago 190.6 MB + 5b9d37953ecd 5 months ago 545.5 MB + 04f05d293573 5 months ago 538.4 MB + 2bb5ecaad6cd 5 months ago 432.1 MB + a73f6ba144e0 5 months ago 432.1 MB +npateriyainsecure/python-hello-world latest 4ba3444051dd 5 months ago 696 MB + a05cc7ed3f32 5 months ago 345.8 MB + 93aeef79b874 5 months ago 432.1 MB + ec4240732685 6 months ago 432.1 MB +registry 2 5dfdbfb4ed57 6 months ago 224.5 MB + 4d0de63b9bfd 6 months ago 432.1 MB + a009751560e3 6 months ago 432.1 MB + ac90607b4f57 6 months ago 432.1 MB +python 3.5 7a7c8d68b039 6 months ago 688.8 MB + 5cbeb1361907 6 months ago 756.6 MB + 5af4b8ad53f2 6 months ago 432.1 MB + 029c7ce4f1ce 6 months ago 432.1 MB + 79138a8bc40f 6 months ago 345.7 MB +elasticsearch 1.5 5a2fb5851392 6 months ago 344.3 MB + d9562f2570b1 6 months ago 312.9 MB + f71f863f39cc 6 months ago 748.1 MB + 386b24223f25 6 months ago 748.1 MB + 8f44a6d9b0c6 6 months ago 748.1 MB + 1772029ccbcb 6 months ago 748.1 MB + 1d035363eb72 6 months ago 748.1 MB + eb3ace28d42f 6 months ago 748.1 MB + a2e12f62e906 6 months ago 455.7 MB + 51764e894e90 6 months ago 190.6 MB + d486de4739e7 6 months ago 431.8 MB + 65ee3eb17e38 6 months ago 432.1 MB + c1ec33d511fd 6 months ago 432.1 MB + 2dccb4dc1278 6 months ago 432.1 MB + 8c49caddfbad 6 months ago 430.3 MB + e744b2d1ecc2 6 months ago 190.6 MB + be8bbb87b68c 6 months ago 190.6 MB + 35a1605a2301 6 months ago 408.7 MB + 99a16d9cc6a3 6 months ago 190.6 MB + b042cf580edb 6 months ago 190.6 MB + 03b086894680 6 months ago 190.6 MB + 5faae5e92c62 6 months ago 190.6 MB + ca616eeaae4c 6 months ago 880.8 MB +gcr.io/google_containers/etcd 2.2.1 fbea2d67e633 7 months ago 28.19 MB +hello-world latest 0a6ba66e537a 8 months ago 960 B +centos 6 3bbbf0aca359 8 months ago 190.6 MB + 501f51238f9e 8 months ago 190.6 MB +gcr.io/google_containers/pause 2.0 8950680a606c 8 months ago 350.2 kB +docker/whalesay latest fb434121fc77 12 months ago 247 MB +steeef/sensu-centos latest b6f14ecbffa0 2 years ago 1.284 GB diff -r 000000000000 -r 26e919a36f86 tests/test_parse.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/test_parse.py Thu Feb 09 09:47:04 2017 -0800 @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +""" +test parsing +""" + +# imports +import speedtest.parse +import os +import shutil +import tempfile +import unittest + +here = os.path.dirname(os.path.abspath(__file__)) + +class ParsingTest(unittest.TestCase): + + def test_parse_columns(self): + """test columns parsing""" + + columns = ['REPOSITORY', + 'TAG', + 'IMAGE ID', + 'CREATED', + 'VIRTUAL SIZE'] + + # load data + output = open(os.path.join(here, 'data', 'docker_images.out')).read() + + # parse the data + data = speedtest.parse.parse_columns(output, columns) + # (Pdb) data[-1] + # {'REPOSITORY': 'steeef/sensu-centos', 'CREATED': '2 years ago', 'IMAGE': 'b6f14e', 'VIRTUAL': '1.284 GB', 'TAG': 'latest', 'ID': 'cbffa0', 'SIZE': ''} + self.assertEqual(data[-1].get("IMAGE ID"), "b6f14ecbffa0") + +if __name__ == '__main__': + unittest.main() diff -r 000000000000 -r 26e919a36f86 tests/testall.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/testall.py Thu Feb 09 09:47:04 2017 -0800 @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +""" +run all unit tests +""" + +import os +import sys +import unittest + +here = os.path.dirname(os.path.abspath(__file__)) + +def main(args=sys.argv[1:]): + + results = unittest.TestResult() + suite = unittest.TestLoader().discover(here, 'test_*.py') + suite.run(results) + n_errors = len(results.errors) + n_failures = len(results.failures) + print ("Run {} tests ({} failures; {} errors)".format(results.testsRun, + n_failures, + n_errors)) + if results.wasSuccessful(): + print ("Success") + sys.exit(0) + else: + # print failures and errors + for label, item in (('FAIL', results.failures), + ('ERROR', results.errors)): + if item: + print ("\n{}::\n".format(label)) + for index, (i, message) in enumerate(item): + print ('{}) {}:'.format(index, str(i))) + print (message) + sys.exit(1) + +if __name__ == '__main__': + main() + +