Mercurial > hg > carton
annotate carton.py @ 18:c6a03199d4bf
stub out package creation; next: to test this
| author | Jeff Hammel <jhammel@mozilla.com> |
|---|---|
| date | Fri, 08 Jul 2011 17:19:25 -0700 |
| parents | b05f5f1ec26e |
| children | 46882eaebb59 |
| rev | line source |
|---|---|
| 0 | 1 #!/usr/bin/env python |
| 2 | |
| 3 """ | |
| 4 make a self-extracting virtualenv from directories or URLs | |
| 5 of packages | |
| 6 | |
| 7 To package up all files in a virtualenvs source directory (e.g.): | |
| 8 | |
| 9 python path/to/carton.py mozmill mozmill/src/* | |
| 10 | |
| 11 This will create a self-extracting file, `mozmill.py`, that will unfold | |
| 12 a virtualenv | |
| 13 """ | |
| 14 | |
| 15 # imports | |
| 16 import os | |
| 17 import sys | |
|
18
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
18 import subprocess |
| 0 | 19 import tarfile |
| 20 import tempfile | |
| 21 import urllib2 | |
| 22 from optparse import OptionParser | |
| 23 from StringIO import StringIO | |
| 24 | |
| 25 # global variables | |
| 26 usage = "%prog [options] environment_name directory|url [...]" | |
| 27 virtualenv_url = 'http://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.6.1.tar.gz' | |
| 28 template = """#!/usr/bin/env python | |
| 29 | |
|
15
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
30 "create a virtualenv at %(ENV)s" |
|
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
31 |
| 0 | 32 import os |
| 33 import shutil | |
| 34 import subprocess | |
| 35 import sys | |
| 36 import tarfile | |
| 37 import tempfile | |
| 38 from optparse import OptionParser | |
| 39 from StringIO import StringIO | |
| 40 | |
| 41 try: | |
| 42 call = subprocess.check_call | |
| 43 except AttributeError: | |
| 44 # old python; boo :( | |
| 45 call = subprocess.call | |
| 46 | |
| 47 # virtualenv name | |
| 48 ENV='''%(ENV)s''' | |
| 49 | |
| 50 # packed files | |
|
3
75919adb199a
use compression, but it doesnt seem to help much
Jeff Hammel <jhammel@mozilla.com>
parents:
0
diff
changeset
|
51 VIRTUAL_ENV='''%(VIRTUAL_ENV)s'''.decode('base64').decode('zlib') |
|
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
52 PACKAGE_SOURCES=%(PACKAGE_SOURCES)s |
| 0 | 53 |
|
15
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
54 # parse options |
|
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
55 usage = os.path.basename(sys.argv[0]) + ' [options]' |
|
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
56 parser = OptionParser(usage=usage, description=__doc__) |
|
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
57 parser.add_option('--env', dest='env', help="environment name [DEFAULT: " + ENV + "]") |
|
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
58 options, args = parser.parse_args() |
|
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
59 if options.env: |
|
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
60 ENV = options.env |
|
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
61 |
| 0 | 62 # unpack virtualenv |
| 63 tempdir = tempfile.mkdtemp() | |
| 64 buffer = StringIO() | |
| 65 buffer.write(VIRTUAL_ENV) | |
| 66 buffer.seek(0) | |
| 67 tf = tarfile.open(mode='r', fileobj=buffer) | |
| 68 tf.extractall(tempdir) | |
| 69 | |
| 70 # find the virtualenv | |
| 71 for root, dirs, files in os.walk(tempdir): | |
| 72 if 'virtualenv.py' in files: | |
| 73 virtualenv = os.path.join(root, 'virtualenv.py') | |
| 74 break | |
| 75 else: | |
| 76 raise Exception("virtualenv.py not found in " + tempdir) | |
| 77 | |
| 78 # create the virtualenv | |
|
17
b05f5f1ec26e
pop PYTHONHOME as this will fuxor things
Jeff Hammel <jhammel@mozilla.com>
parents:
16
diff
changeset
|
79 os.environ.pop('PYTHONHOME', None) |
| 0 | 80 call([sys.executable, virtualenv, ENV]) |
| 81 | |
| 82 # find the bin/scripts directory | |
| 83 for i in ('bin', 'Scripts'): | |
| 84 scripts_dir = os.path.abspath(os.path.join(ENV, i)) | |
| 85 if os.path.exists(scripts_dir): | |
| 86 break | |
| 87 else: | |
| 88 raise Exception("Scripts directory not found in " + ENV) | |
| 89 | |
| 90 # find the virtualenv's python | |
| 91 for i in ('python', 'python.exe'): | |
| 92 python = os.path.join(scripts_dir, i) | |
| 93 if os.path.exists(python): | |
| 94 break | |
| 95 else: | |
| 96 raise Exception("python not found in " + scripts_dir) | |
| 97 | |
|
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
98 # unpack the sources and setup for development |
|
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
99 srcdir = os.path.join(ENV, 'src') |
|
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
100 os.mkdir(srcdir) |
|
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
101 setup_pys = set() |
|
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
102 for source in PACKAGE_SOURCES: |
|
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
103 source = source.decode('base64').decode('zlib') |
|
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
104 buffer = StringIO() |
|
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
105 buffer.write(source) |
|
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
106 buffer.seek(0) |
|
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
107 tf = tarfile.open(mode='r', fileobj=buffer) |
|
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
108 tf.extractall(srcdir) |
| 0 | 109 |
|
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
110 # setup sources for development if there are any new setup.py files |
| 16 | 111 # TODO: ideally this would figure out dependency order for you |
|
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
112 for i in os.listdir(srcdir): |
|
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
113 if i in setup_pys: |
|
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
114 continue |
|
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
115 subdir = os.path.join(srcdir, i) |
|
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
116 if os.path.exists(os.path.join(srcdir, i, 'setup.py')): |
|
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
117 call([python, 'setup.py', 'develop'], cwd=subdir) |
|
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
118 setup_pys.add(i) |
| 0 | 119 |
|
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
120 # cleanup tempdir # TODO (optionally?) |
| 0 | 121 # shutil.rmtree(tempdir) |
| 122 | |
| 123 # TODO: | |
| 124 # - add carton to the virtualenv (!) | |
| 125 # - add virtualenv to the virtualenv (!) | |
| 126 """ | |
| 127 | |
| 128 def isURL(path): | |
| 129 return path.startswith('http://') or path.startswith('https://') | |
| 130 | |
|
18
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
131 try: |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
132 call = subprocess.check_call |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
133 except AttributeError: |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
134 # old python; boo :( |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
135 call = subprocess.call |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
136 |
| 0 | 137 def main(args=sys.argv[1:]): |
| 138 | |
| 139 # parse CLI arguments | |
| 140 parser = OptionParser(usage=usage, description=__doc__) | |
| 141 parser.add_option('-o', dest='outfile', | |
| 142 help="specify outfile; otherwise it will come from environment_name") | |
|
18
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
143 parser.add_option('-p', '--package', dest='package', |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
144 action='store_true', default=False, |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
145 help="create python packages from sources; do not take entire subdirectory") |
| 0 | 146 parser.add_option('--virtualenv', dest='virtualenv', |
|
12
542b46ac4e28
fix description more better
Jeff Hammel <jhammel@mozilla.com>
parents:
11
diff
changeset
|
147 help="use this virtualenv URL or file tarball") |
| 0 | 148 options, args = parser.parse_args(args) |
| 149 if len(args) < 2: | |
| 150 parser.print_usage() | |
| 151 parser.exit() | |
| 152 environment = args[0] | |
| 153 if environment.endswith('.py'): | |
| 154 # stop on .py; will add it in later | |
| 155 environment = environment[:-3] | |
| 156 sources = args[1:] | |
| 157 | |
| 158 # tar up the sources | |
|
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
159 source_array = [] |
| 0 | 160 for source in sources: |
|
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
161 buffer = None |
| 0 | 162 |
| 163 if isURL(source): | |
|
14
c474362cf69a
allow fetching of remote tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
164 # remote tarball or resource |
|
c474362cf69a
allow fetching of remote tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
165 buffer = urllib2.urlopen(source).read() |
| 0 | 166 else: |
|
18
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
167 # local directory or tarball |
|
13
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
168 assert os.path.exists(source), "%s does not exist" % source |
|
18
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
169 |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
170 # package up the source if applicable |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
171 if options.package and os.path.exists(os.path.join(source, 'setup.py')): |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
172 call([sys.executable, 'setup.py', 'sdist'], cwd=source) |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
173 dist_dir = os.path.join(source, 'dist') |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
174 assert os.path.isdir(dist_dir), "dist directory not created in %s" % source |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
175 tarfiles = [i for i in os.listdir(dist_dir) |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
176 if i.endswith('.tar.gz')] |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
177 assert tarfiles, "no .tar.gz files found in %s" % dist_dir |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
178 def last_modified(filename): |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
179 return os.path.getmtime(os.path.join(dist_dir, filename)) |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
180 tarfiles.sort(key=last_modified) |
|
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
181 source = os.path.join(dist_dir, tarfiles[-1]) |
|
13
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
182 |
|
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
183 if (not os.path.isdir(source)) and tarfile.is_tarfile(source): |
|
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
184 # check for a tarball |
|
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
185 buffer = file(source).read() |
|
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
186 else: |
|
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
187 source_buffer = StringIO() |
|
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
188 source_tar = tarfile.open(mode="w:gz", fileobj=source_buffer) |
|
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
189 source_tar.add(source, arcname=os.path.basename(source)) |
|
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
190 source_tar.close() |
|
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
191 buffer = source_buffer.getvalue() |
|
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
192 |
| 0 | 193 # could use git, hg, etc repos. but probably shouldn't |
|
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
194 |
|
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
195 source_array.append(buffer.encode('zlib').encode('base64')) |
| 0 | 196 |
| 197 # tar up virtualenv if not available | |
| 198 if options.virtualenv: | |
| 199 if isURL(options.virtualenv): | |
| 200 globals()['VIRTUAL_ENV'] = urllib2.urlopen(options.virtualenv).read() | |
| 201 else: | |
| 202 assert os.path.exists(options.virtualenv) | |
| 203 if os.path.isdir(options.virtualenv): | |
| 204 raise NotImplementedError("Hypothetically you should be able to use a local directory or tarball, but I haven't done this yet") | |
| 205 else: | |
| 206 # assert a tarfile | |
| 207 assert tarfile.is_tarfile(options.virtualenv), "%s must be a tar file" % options.virtualenv | |
| 208 globals()['VIRTUAL_ENV'] = file(options.virtualenv).read() | |
| 209 else: | |
| 210 globals()['VIRTUAL_ENV'] = urllib2.urlopen(virtualenv_url).read() | |
| 211 # TODO: used the below hashed value of VIRTUAL_ENV if set | |
| 212 # (set that with another file) | |
| 213 | |
| 214 # interpolate "template" -> output | |
| 215 outfile = options.outfile | |
| 216 if outfile is None: | |
| 217 outfile = environment + '.py' | |
|
3
75919adb199a
use compression, but it doesnt seem to help much
Jeff Hammel <jhammel@mozilla.com>
parents:
0
diff
changeset
|
218 variables = {'VIRTUAL_ENV': VIRTUAL_ENV.encode('zlib').encode('base64'), |
| 0 | 219 'ENV': environment, |
|
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
220 'PACKAGE_SOURCES': repr(source_array)} |
| 0 | 221 f = file(outfile, 'w') |
| 222 f.write(template % variables) | |
| 223 f.close() | |
| 224 try: | |
| 225 os.chmod(outfile, 0755) | |
| 226 except: | |
| 227 # you probably don't have os.chmod | |
| 228 pass | |
| 229 | |
| 230 VIRTUAL_ENV = """""" | |
| 231 | |
| 232 if __name__ == '__main__': | |
| 233 main() |