annotate lemuriformes/csv2sql.py @ 15:0d1b8bb1d97b

SQL + data related functionality
author Jeff Hammel <k0scist@gmail.com>
date Sun, 10 Dec 2017 17:16:52 -0800
parents 82cd4e0b66cf
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
12
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
1 #!/usr/bin/env python
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
2
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
3 """
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
4 convert CSV to SQL
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
5 """
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
6 # (whitepaper)
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
7
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
8 import sys
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
9 from .cast import infer
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
10 from .cli import ConfigurationParser
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
11 from .columns import read_columns
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
12 from .url2sql import url2sql
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
13
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
14 def main(args=sys.argv[1:]):
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
15 """CLI"""
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
16
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
17 # parse command line
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
18 parser = ConfigurationParser(description=__doc__)
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
19 parser.add_argument('csv',
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
20 type=read_columns,
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
21 help="CSV source to read")
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
22 parser.add_argument('sql',
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
23 type=url2sql,
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
24 help="SQL connection URL")
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
25 parser.add_argument('table', # TODO: this would be nice to have as part of the URL </sugar>
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
26 help="SQL table to create")
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
27 options = parser.parse_args(args)
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
28
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
29 # infer column types
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
30 columns = options.csv.keys()
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
31 column_types = {column: infer(options.csv[column])
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
32 for column in columns}
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
33
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
34 # cast columns
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
35 for column in columns:
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
36 _type = column_types[column]
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
37 options.csv[column] = [_type(v) for v in options.csv[column]]
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
38
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
39 # TODO:
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
40 # - create table(s)
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
41 # - populate data
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
42
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
43
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
44 if __name__ == '__main__':
82cd4e0b66cf csv2sql
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
45 main()