Mercurial > hg > carton
annotate carton.py @ 22:9c710f06e51d
add virtualenv to the virtualenv(!)
author | Jeff Hammel <jhammel@mozilla.com> |
---|---|
date | Fri, 08 Jul 2011 18:11:09 -0700 |
parents | 46882eaebb59 |
children | 987086aad234 |
rev | line source |
---|---|
0 | 1 #!/usr/bin/env python |
2 | |
3 """ | |
4 make a self-extracting virtualenv from directories or URLs | |
5 of packages | |
6 | |
7 To package up all files in a virtualenvs source directory (e.g.): | |
8 | |
9 python path/to/carton.py mozmill mozmill/src/* | |
10 | |
11 This will create a self-extracting file, `mozmill.py`, that will unfold | |
12 a virtualenv | |
13 """ | |
14 | |
15 # imports | |
16 import os | |
17 import sys | |
18
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
18 import subprocess |
0 | 19 import tarfile |
20 import tempfile | |
21 import urllib2 | |
22 from optparse import OptionParser | |
23 from StringIO import StringIO | |
24 | |
25 # global variables | |
26 usage = "%prog [options] environment_name directory|url [...]" | |
27 virtualenv_url = 'http://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.6.1.tar.gz' | |
28 template = """#!/usr/bin/env python | |
29 | |
15
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
30 "create a virtualenv at %(ENV)s" |
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
31 |
0 | 32 import os |
33 import shutil | |
34 import subprocess | |
35 import sys | |
36 import tarfile | |
37 import tempfile | |
38 from optparse import OptionParser | |
39 from StringIO import StringIO | |
40 | |
41 try: | |
42 call = subprocess.check_call | |
43 except AttributeError: | |
44 # old python; boo :( | |
45 call = subprocess.call | |
46 | |
47 # virtualenv name | |
48 ENV='''%(ENV)s''' | |
49 | |
50 # packed files | |
3
75919adb199a
use compression, but it doesnt seem to help much
Jeff Hammel <jhammel@mozilla.com>
parents:
0
diff
changeset
|
51 VIRTUAL_ENV='''%(VIRTUAL_ENV)s'''.decode('base64').decode('zlib') |
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
52 PACKAGE_SOURCES=%(PACKAGE_SOURCES)s |
0 | 53 |
15
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
54 # parse options |
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
55 usage = os.path.basename(sys.argv[0]) + ' [options]' |
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
56 parser = OptionParser(usage=usage, description=__doc__) |
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
57 parser.add_option('--env', dest='env', help="environment name [DEFAULT: " + ENV + "]") |
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
58 options, args = parser.parse_args() |
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
59 if options.env: |
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
60 ENV = options.env |
f05e636b7444
now you can specify the name of the env in the created script
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
61 |
0 | 62 # unpack virtualenv |
63 tempdir = tempfile.mkdtemp() | |
64 buffer = StringIO() | |
65 buffer.write(VIRTUAL_ENV) | |
66 buffer.seek(0) | |
67 tf = tarfile.open(mode='r', fileobj=buffer) | |
68 tf.extractall(tempdir) | |
69 | |
70 # find the virtualenv | |
71 for root, dirs, files in os.walk(tempdir): | |
72 if 'virtualenv.py' in files: | |
73 virtualenv = os.path.join(root, 'virtualenv.py') | |
74 break | |
75 else: | |
76 raise Exception("virtualenv.py not found in " + tempdir) | |
77 | |
78 # create the virtualenv | |
17
b05f5f1ec26e
pop PYTHONHOME as this will fuxor things
Jeff Hammel <jhammel@mozilla.com>
parents:
16
diff
changeset
|
79 os.environ.pop('PYTHONHOME', None) |
0 | 80 call([sys.executable, virtualenv, ENV]) |
81 | |
82 # find the bin/scripts directory | |
83 for i in ('bin', 'Scripts'): | |
84 scripts_dir = os.path.abspath(os.path.join(ENV, i)) | |
85 if os.path.exists(scripts_dir): | |
86 break | |
87 else: | |
88 raise Exception("Scripts directory not found in " + ENV) | |
89 | |
90 # find the virtualenv's python | |
91 for i in ('python', 'python.exe'): | |
92 python = os.path.join(scripts_dir, i) | |
93 if os.path.exists(python): | |
94 break | |
95 else: | |
96 raise Exception("python not found in " + scripts_dir) | |
97 | |
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
98 # unpack the sources and setup for development |
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
99 srcdir = os.path.join(ENV, 'src') |
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
100 os.mkdir(srcdir) |
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
101 setup_pys = set() |
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
102 for source in PACKAGE_SOURCES: |
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
103 source = source.decode('base64').decode('zlib') |
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
104 buffer = StringIO() |
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
105 buffer.write(source) |
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
106 buffer.seek(0) |
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
107 tf = tarfile.open(mode='r', fileobj=buffer) |
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
108 tf.extractall(srcdir) |
0 | 109 |
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
110 # setup sources for development if there are any new setup.py files |
16 | 111 # TODO: ideally this would figure out dependency order for you |
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
112 for i in os.listdir(srcdir): |
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
113 if i in setup_pys: |
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
114 continue |
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
115 subdir = os.path.join(srcdir, i) |
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
116 if os.path.exists(os.path.join(srcdir, i, 'setup.py')): |
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
117 call([python, 'setup.py', 'develop'], cwd=subdir) |
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
118 setup_pys.add(i) |
0 | 119 |
22
9c710f06e51d
add virtualenv to the virtualenv(!)
Jeff Hammel <jhammel@mozilla.com>
parents:
21
diff
changeset
|
120 # add virtualenv to the virtualenv (!) |
9c710f06e51d
add virtualenv to the virtualenv(!)
Jeff Hammel <jhammel@mozilla.com>
parents:
21
diff
changeset
|
121 virtualenv_dir = os.path.dirname(virtualenv) |
9c710f06e51d
add virtualenv to the virtualenv(!)
Jeff Hammel <jhammel@mozilla.com>
parents:
21
diff
changeset
|
122 if os.path.exists(os.path.join(virtualenv_dir, 'setup.py')): |
9c710f06e51d
add virtualenv to the virtualenv(!)
Jeff Hammel <jhammel@mozilla.com>
parents:
21
diff
changeset
|
123 call([python, 'setup.py', 'install'], cwd=virtualenv_dir) |
9c710f06e51d
add virtualenv to the virtualenv(!)
Jeff Hammel <jhammel@mozilla.com>
parents:
21
diff
changeset
|
124 |
0 | 125 # TODO: |
126 # - add carton to the virtualenv (!) | |
21
46882eaebb59
consolidate and order TODO for generated script
Jeff Hammel <jhammel@mozilla.com>
parents:
18
diff
changeset
|
127 # - cleanup tempdir |
46882eaebb59
consolidate and order TODO for generated script
Jeff Hammel <jhammel@mozilla.com>
parents:
18
diff
changeset
|
128 # shutil.rmtree(tempdir) |
0 | 129 """ |
130 | |
131 def isURL(path): | |
132 return path.startswith('http://') or path.startswith('https://') | |
133 | |
18
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
134 try: |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
135 call = subprocess.check_call |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
136 except AttributeError: |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
137 # old python; boo :( |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
138 call = subprocess.call |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
139 |
0 | 140 def main(args=sys.argv[1:]): |
141 | |
142 # parse CLI arguments | |
143 parser = OptionParser(usage=usage, description=__doc__) | |
144 parser.add_option('-o', dest='outfile', | |
145 help="specify outfile; otherwise it will come from environment_name") | |
18
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
146 parser.add_option('-p', '--package', dest='package', |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
147 action='store_true', default=False, |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
148 help="create python packages from sources; do not take entire subdirectory") |
0 | 149 parser.add_option('--virtualenv', dest='virtualenv', |
12
542b46ac4e28
fix description more better
Jeff Hammel <jhammel@mozilla.com>
parents:
11
diff
changeset
|
150 help="use this virtualenv URL or file tarball") |
0 | 151 options, args = parser.parse_args(args) |
152 if len(args) < 2: | |
153 parser.print_usage() | |
154 parser.exit() | |
155 environment = args[0] | |
156 if environment.endswith('.py'): | |
157 # stop on .py; will add it in later | |
158 environment = environment[:-3] | |
159 sources = args[1:] | |
160 | |
161 # tar up the sources | |
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
162 source_array = [] |
0 | 163 for source in sources: |
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
164 buffer = None |
0 | 165 |
166 if isURL(source): | |
14
c474362cf69a
allow fetching of remote tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
167 # remote tarball or resource |
c474362cf69a
allow fetching of remote tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
13
diff
changeset
|
168 buffer = urllib2.urlopen(source).read() |
0 | 169 else: |
18
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
170 # local directory or tarball |
13
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
171 assert os.path.exists(source), "%s does not exist" % source |
18
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
172 |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
173 # package up the source if applicable |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
174 if options.package and os.path.exists(os.path.join(source, 'setup.py')): |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
175 call([sys.executable, 'setup.py', 'sdist'], cwd=source) |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
176 dist_dir = os.path.join(source, 'dist') |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
177 assert os.path.isdir(dist_dir), "dist directory not created in %s" % source |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
178 tarfiles = [i for i in os.listdir(dist_dir) |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
179 if i.endswith('.tar.gz')] |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
180 assert tarfiles, "no .tar.gz files found in %s" % dist_dir |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
181 def last_modified(filename): |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
182 return os.path.getmtime(os.path.join(dist_dir, filename)) |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
183 tarfiles.sort(key=last_modified) |
c6a03199d4bf
stub out package creation; next: to test this
Jeff Hammel <jhammel@mozilla.com>
parents:
17
diff
changeset
|
184 source = os.path.join(dist_dir, tarfiles[-1]) |
13
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
185 |
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
186 if (not os.path.isdir(source)) and tarfile.is_tarfile(source): |
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
187 # check for a tarball |
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
188 buffer = file(source).read() |
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
189 else: |
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
190 source_buffer = StringIO() |
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
191 source_tar = tarfile.open(mode="w:gz", fileobj=source_buffer) |
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
192 source_tar.add(source, arcname=os.path.basename(source)) |
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
193 source_tar.close() |
f522620c6a78
now works properly with tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
194 buffer = source_buffer.getvalue() |
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
195 |
0 | 196 # could use git, hg, etc repos. but probably shouldn't |
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
197 |
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
198 source_array.append(buffer.encode('zlib').encode('base64')) |
0 | 199 |
200 # tar up virtualenv if not available | |
201 if options.virtualenv: | |
202 if isURL(options.virtualenv): | |
203 globals()['VIRTUAL_ENV'] = urllib2.urlopen(options.virtualenv).read() | |
204 else: | |
205 assert os.path.exists(options.virtualenv) | |
206 if os.path.isdir(options.virtualenv): | |
207 raise NotImplementedError("Hypothetically you should be able to use a local directory or tarball, but I haven't done this yet") | |
208 else: | |
209 # assert a tarfile | |
210 assert tarfile.is_tarfile(options.virtualenv), "%s must be a tar file" % options.virtualenv | |
211 globals()['VIRTUAL_ENV'] = file(options.virtualenv).read() | |
212 else: | |
213 globals()['VIRTUAL_ENV'] = urllib2.urlopen(virtualenv_url).read() | |
214 # TODO: used the below hashed value of VIRTUAL_ENV if set | |
215 # (set that with another file) | |
216 | |
217 # interpolate "template" -> output | |
218 outfile = options.outfile | |
219 if outfile is None: | |
220 outfile = environment + '.py' | |
3
75919adb199a
use compression, but it doesnt seem to help much
Jeff Hammel <jhammel@mozilla.com>
parents:
0
diff
changeset
|
221 variables = {'VIRTUAL_ENV': VIRTUAL_ENV.encode('zlib').encode('base64'), |
0 | 222 'ENV': environment, |
11
e6a62ba0c24d
now respect the order sources are installed in
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
223 'PACKAGE_SOURCES': repr(source_array)} |
0 | 224 f = file(outfile, 'w') |
225 f.write(template % variables) | |
226 f.close() | |
227 try: | |
228 os.chmod(outfile, 0755) | |
229 except: | |
230 # you probably don't have os.chmod | |
231 pass | |
232 | |
233 VIRTUAL_ENV = """""" | |
234 | |
235 if __name__ == '__main__': | |
236 main() |