Mercurial > mozilla > hg > DocumentIt
annotate document_it.py @ 23:e6ed732e8ce6
now works with files anyway
author | Jeff Hammel <jhammel@mozilla.com> |
---|---|
date | Wed, 03 Aug 2011 12:15:50 -0700 |
parents | 67e4becc7d49 |
children | 045544951756 |
rev | line source |
---|---|
0 | 1 #!/usr/bin/env python |
2 | |
3 """ | |
9 | 4 update MDN documentation from markdown |
4
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
5 |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
6 see: |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
7 http://developer.mindtouch.com/en/ref/MindTouch_API/POST%3Apages%2F%2F%7Bpageid%7D%2F%2Fcontents |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
8 |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
9 The manifest format is in the form: |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
10 |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
11 mozrunner/README.txt https://developer.mozilla.org/en/Mozrunner |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
12 jsbridge/README.txt https://developer.mozilla.org/en/JSbridge |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
13 mozmill/README.txt https://developer.mozilla.org/en/Mozmill |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
14 mozmill/docs/ https://developer.mozilla.org/en/Mozmill/ |
9 | 15 |
16 --dest sets the destination, e.g. | |
17 | |
18 --dest http://developer.mozilla.org/ | |
19 --dest https://developer-stage9.mozilla.org/jhammel | |
20 --dest path/to directory | |
21 | |
22 By default, a new temporary directory will be created | |
0 | 23 """ |
24 | |
5
3464eda1af80
better formatting for command line
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
25 import optparse |
0 | 26 import os |
27 import sys | |
9 | 28 import tempfile |
0 | 29 import urllib2 |
30 | |
22
67e4becc7d49
first steps for cleaning up this awful mess
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
31 # import markdown |
0 | 32 try: |
33 import markdown | |
34 except ImportError: | |
35 raise ImportError("markdown is not installed, run (e.g.):\neasy_install Markdown") | |
36 | |
23
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
37 destinations = {'stage': 'https://developer-stage9.mozilla.org/@api/deki/pages/=%(page)s/contents?edittime=now', |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
38 'MDN': 'https://developer.mozilla.org/@api/deki/pages/=%(page)s/contents?edittime=now'} |
22
67e4becc7d49
first steps for cleaning up this awful mess
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
39 |
0 | 40 |
41 def find_readme(directory): | |
42 """find a README file in a directory""" | |
4
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
43 # XXX currently unused |
5
3464eda1af80
better formatting for command line
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
44 README=['README.md', 'README.txt', 'README'] |
0 | 45 for name in README: |
46 path = os.path.join(directory, name) | |
47 if os.path.exists(path): | |
48 return path | |
49 | |
18 | 50 def all_files(directory): |
22
67e4becc7d49
first steps for cleaning up this awful mess
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
51 """get all files in a directory tree""" |
18 | 52 filenames = [] |
53 for dirpath, dirnames, files in os.walk(directory): | |
54 filenames.extend([os.path.join(dirpath, f) for f in files]) | |
55 return sorted(filenames) | |
56 | |
22
67e4becc7d49
first steps for cleaning up this awful mess
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
57 def parse_manifest(filename, directory=None): |
4
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
58 """ |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
59 reads a documentation manifest; returns a list of two-tuples: |
6 | 60 [(filename, destination)] |
4
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
61 """ |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
62 |
6 | 63 assert os.path.exists(filename) and os.path.isfile(filename), "%s not found" % filename |
64 | |
4
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
65 if directory is None: |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
66 directory = os.path.dirname(os.path.abspath(filename)) |
5
3464eda1af80
better formatting for command line
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
67 lines = [line.strip() for line in file(filename).readlines()] |
4
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
68 lines = [line for line in lines |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
69 if line and not line.startswith('#')] |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
70 items = [] |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
71 for line in lines: |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
72 try: |
22
67e4becc7d49
first steps for cleaning up this awful mess
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
73 f, page = line.split() |
4
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
74 # TODO: include options as third segment (e.g. format=ReST) |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
75 except ValueError: |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
76 raise ValueError("illegal manifest line: '%s'" % line) |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
77 |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
78 filename = os.path.join(directory, f) |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
79 if os.path.isdir(filename): |
22
67e4becc7d49
first steps for cleaning up this awful mess
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
80 raise NotImplementedError |
18 | 81 files = all_files(filename) |
82 for i in files: | |
83 relpath = os.path.relpath(i, filename) | |
22
67e4becc7d49
first steps for cleaning up this awful mess
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
84 items.append((i, relpath)) |
4
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
85 else: |
22
67e4becc7d49
first steps for cleaning up this awful mess
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
86 items.append((filename, page)) |
4
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
87 return items |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
88 |
23
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
89 def item_url(item, dest): |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
90 if '://' in dest: |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
91 if '%(page)s' in dest: |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
92 return dest % {'page': item} |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
93 else: |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
94 return '%s/%s' % (dest.lstrip('/'), item.rstrip('/')) |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
95 else: |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
96 return 'file://%s' % (os.path.join(dest, item)) |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
97 |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
98 |
0 | 99 def main(args=sys.argv[1:]): |
100 | |
9 | 101 # default output directory |
102 default_dir = tempfile.mktemp() | |
103 | |
0 | 104 # parse command line options |
4
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
105 usage = '%prog [options] manifest <manifest> <...>' |
5
3464eda1af80
better formatting for command line
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
106 |
3464eda1af80
better formatting for command line
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
107 # description formatter |
3464eda1af80
better formatting for command line
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
108 class PlainDescriptionFormatter(optparse.IndentedHelpFormatter): |
3464eda1af80
better formatting for command line
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
109 def format_description(self, description): |
3464eda1af80
better formatting for command line
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
110 if description: |
13
1efb669ab1c8
better description formatting
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
111 return description.strip() + '\n' |
5
3464eda1af80
better formatting for command line
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
112 else: |
3464eda1af80
better formatting for command line
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
113 return '' |
3464eda1af80
better formatting for command line
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
114 |
3464eda1af80
better formatting for command line
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
115 parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter()) |
0 | 116 parser.add_option('-d', '--directory', dest='directory', |
6 | 117 help='render the documentation from this directory') |
15
d9026d114655
deal with eccentricities of local directories
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
118 parser.add_option('-o', '--dest', dest='dest', |
9 | 119 default=default_dir, |
120 help='base directory or URL of destination [DEFAULT: %default]') | |
4
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
121 parser.add_option('-u', '--user', dest='user', |
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
122 help='user name') |
20 | 123 parser.add_option('-p', '--password', dest='password', |
124 help='user password') | |
6 | 125 parser.add_option('--list', dest='list', action='store_true', default=False, |
126 help="list files") | |
9 | 127 parser.add_option('--validate', dest='validate', # TODO unused |
128 action='store_true', default=False, | |
129 help="validate the rendering but don't output") | |
4
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
130 options, manifests = parser.parse_args(args) |
0 | 131 |
6 | 132 # print help if no manifests given |
4
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
133 if not args: |
5
3464eda1af80
better formatting for command line
Jeff Hammel <jhammel@mozilla.com>
parents:
4
diff
changeset
|
134 parser.print_help() |
4
550f4b240e20
make almost look like what we want to do
Jeff Hammel <jhammel@mozilla.com>
parents:
2
diff
changeset
|
135 parser.exit() |
0 | 136 |
7 | 137 # get base url |
11 | 138 assert options.dest |
23
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
139 if options.dest.startswith('file://'): |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
140 options.dest = options.dest[len('file://'):] |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
141 if '://' not in options.dest: |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
142 options.dest = os.path.abspath(options.dest) |
7 | 143 |
6 | 144 # read the manifests |
145 files = [] | |
146 for manifest in manifests: | |
22
67e4becc7d49
first steps for cleaning up this awful mess
Jeff Hammel <jhammel@mozilla.com>
parents:
20
diff
changeset
|
147 for item in parse_manifest(manifest, options.directory): |
6 | 148 if item not in files: |
149 files.append(item) | |
23
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
150 files = [(i, item_url(j, options.dest)) for i, j in files] |
6 | 151 if options.list: |
152 for item in files: | |
14 | 153 print '%s -> %s' % item |
17
0a1aecef2c52
youre done after listing files
Jeff Hammel <jhammel@mozilla.com>
parents:
16
diff
changeset
|
154 return |
0 | 155 |
9 | 156 if not files: |
157 return # you're done | |
158 | |
0 | 159 # render and upload READMEs |
23
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
160 if '://' in options.dest: |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
161 |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
162 # check credentials |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
163 assert options.user and options.password, "Please supply your --user and --password" |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
164 |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
165 # upload the files |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
166 # XXX just use curl for now |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
167 import subprocess |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
168 for src, dest in files: |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
169 dest += '/contents?edittime=now' |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
170 print src, dest |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
171 |
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
172 else: |
8 | 173 |
11 | 174 # ensure a directory |
175 if os.path.exists(options.dest): | |
176 assert os.path.isdir(options.dest), "'%s' - not a directory" % options.dest | |
8 | 177 |
23
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
178 # render to directory |
9 | 179 for src, dest in files: |
15
d9026d114655
deal with eccentricities of local directories
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
180 |
d9026d114655
deal with eccentricities of local directories
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
181 if dest.startswith('file://'): |
d9026d114655
deal with eccentricities of local directories
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
182 dest = dest[len('file://'):] |
11 | 183 |
184 # create a directory if needed | |
185 dirname = os.path.dirname(dest) | |
16
d6528dd74592
only create the directory if we need to
Jeff Hammel <jhammel@mozilla.com>
parents:
15
diff
changeset
|
186 if os.path.exists(dirname): |
19
493683e17eae
now renders correctly to filesystem for preview
Jeff Hammel <jhammel@mozilla.com>
parents:
18
diff
changeset
|
187 if not os.path.isdir(dirname): |
493683e17eae
now renders correctly to filesystem for preview
Jeff Hammel <jhammel@mozilla.com>
parents:
18
diff
changeset
|
188 # deal with filesystem directories vs PATH_INFO |
493683e17eae
now renders correctly to filesystem for preview
Jeff Hammel <jhammel@mozilla.com>
parents:
18
diff
changeset
|
189 f = file(dirname) |
493683e17eae
now renders correctly to filesystem for preview
Jeff Hammel <jhammel@mozilla.com>
parents:
18
diff
changeset
|
190 buffer = f.read() |
493683e17eae
now renders correctly to filesystem for preview
Jeff Hammel <jhammel@mozilla.com>
parents:
18
diff
changeset
|
191 f.close() |
493683e17eae
now renders correctly to filesystem for preview
Jeff Hammel <jhammel@mozilla.com>
parents:
18
diff
changeset
|
192 os.remove(dirname) |
493683e17eae
now renders correctly to filesystem for preview
Jeff Hammel <jhammel@mozilla.com>
parents:
18
diff
changeset
|
193 os.makedirs(dirname) |
493683e17eae
now renders correctly to filesystem for preview
Jeff Hammel <jhammel@mozilla.com>
parents:
18
diff
changeset
|
194 f = file(os.path.join(dirname, 'index.html'), 'w') |
493683e17eae
now renders correctly to filesystem for preview
Jeff Hammel <jhammel@mozilla.com>
parents:
18
diff
changeset
|
195 f.write(buffer) |
493683e17eae
now renders correctly to filesystem for preview
Jeff Hammel <jhammel@mozilla.com>
parents:
18
diff
changeset
|
196 f.close() |
16
d6528dd74592
only create the directory if we need to
Jeff Hammel <jhammel@mozilla.com>
parents:
15
diff
changeset
|
197 else: |
d6528dd74592
only create the directory if we need to
Jeff Hammel <jhammel@mozilla.com>
parents:
15
diff
changeset
|
198 os.makedirs(dirname) |
11 | 199 |
200 # render | |
201 f = file(dest, 'w') | |
202 buffer = markdown.Markdown().convert(file(src).read()) | |
203 f.write(buffer) | |
204 f.close() | |
20 | 205 |
23
e6ed732e8ce6
now works with files anyway
Jeff Hammel <jhammel@mozilla.com>
parents:
22
diff
changeset
|
206 # print out destination directory if using the temporary default |
10 | 207 if options.dest == default_dir: |
11 | 208 print "Files rendered to\n%s" % default_dir |
9 | 209 |
0 | 210 if __name__ == '__main__': |
211 main() |