Skip to content

Commit 49c423e

Browse files
author
visualskyrim
committed
Merge branch 'release/201601061721'
2 parents ae257b5 + a5b7f56 commit 49c423e

8 files changed

Lines changed: 297 additions & 29 deletions

File tree

‎README.md‎

Lines changed: 61 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,15 @@ To login, use following command and follow the instruction given by the tool:
3232

3333
***Step 1***: Get the source code:
3434

35-
`git clone git@github.com:visualskyrim/bquick.git`
35+
```
36+
git clone git@github.com:visualskyrim/bquick.git
37+
```
3638

3739
***Step 2***: Install
3840

39-
`python setup.py install`
41+
```
42+
python setup.py install
43+
```
4044

4145

4246
## Usage
@@ -45,22 +49,28 @@ To login, use following command and follow the instruction given by the tool:
4549

4650
***List all tables:***
4751

48-
`bquick <dataset> ls [-l <rows-to-show>]`
52+
```
53+
bquick <dataset> ls [-l <rows-to-show>]
54+
```
4955

5056
```
5157
bquick test_dateset ls -l 20
5258
```
5359

54-
> The `<rows-to-show>` is defaultly 50
60+
> The `<rows-to-show>` is default 50
5561
5662

5763
***List tables that match wildcard prefix***
5864

59-
`bquick <dataset> ls -w <wildcard-prefix> [-l <rows-to-show>]`
65+
```
66+
bquick <dataset> ls -w <wildcard-prefix> [-l <rows-to-show>]
67+
```
6068

6169
Get all the tables with given wildcard prefix.
6270

63-
`bquick <dataset> ls -w <wildcard-prefix> <start-date> <end-date> [-l <rows-to-show>]`
71+
```
72+
bquick <dataset> ls -w <wildcard-prefix> <start-date> <end-date> [-l <rows-to-show>]
73+
```
6474

6575
Get all the table with given wildcard and within the given date range.
6676

@@ -82,3 +92,48 @@ bquick test_dataset ls -r 'test_table_\w{32}\d{8}'
8292
```
8393

8494
> The `<reg-string>` should be enclosed within single-quote.
95+
96+
### Delete Table
97+
98+
Delete tables in BigQuery.
99+
100+
All the tables to be deleted will be confirmed before the deletion.
101+
102+
***Delete table by name:***
103+
104+
```
105+
bquick <dataset> del -n <table-name>
106+
```
107+
108+
Delete the table with given name.
109+
110+
111+
***Delete tables in the file***
112+
113+
```
114+
bquick <dataset> del -f <tables-file>
115+
```
116+
117+
All the tables with the name in given file will be deleted.
118+
Table name should be one line in the file.
119+
120+
> `<tables-file>` could be either absolute path or relative path pointing to the file.
121+
122+
123+
***Delete tables matching the regex expression:***
124+
125+
```
126+
bquick <dataset> del -r <table-name-pattern>
127+
```
128+
129+
Delete the tables with the name that matches the regex expression.
130+
131+
> `<table-name-pattern>` should be enclosed by single-quote.
132+
133+
***Delete tables matching the wildcard prefix***
134+
135+
```
136+
bquick <dataset> del -w <wildcard-prefix> <start-date> <end-date>
137+
```
138+
139+
Delete all the tables that match `<wildcard-prefix>` and between the `<start-date>` and `<end-date>`.

‎bquick/__init__.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# -*- coding: utf-8 -*-
22

33
from bquick.command_parser import get_command
4-
from bquick.bigquery import list_table
4+
from bquick.bigquery import list_table, delete_table
55

66
def main():
77

@@ -14,6 +14,8 @@ def main():
1414
tables = list_table(command)
1515
for table_name in tables:
1616
print table_name
17+
elif mode == 'del':
18+
delete_table(command)
1719

1820
except Exception as e:
1921
raise e

‎bquick/bigquery/__init__.py‎

Lines changed: 75 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,22 @@
44
import urllib
55
import uuid
66
import time
7+
import os
78
import json
9+
import sys
810
from apiclient.discovery import build
911
from oauth2client.client import GoogleCredentials
1012
from urllib2 import HTTPError
1113

12-
from bquick.bigquery import table_list_handler
14+
from bquick.bigquery import table_list_handler, table_delete_handler
1315
from bquick.command_parser import ListCommand, \
1416
ListRegexCommand, \
15-
ListWildcardCommand
17+
ListWildcardCommand, \
18+
DeleteFileCommand, \
19+
DeleteNamedCommand, \
20+
DeleteRegexCommand, \
21+
DeleteWildcardCommand
22+
1623

1724
# Retry transport and file IO errors.
1825
RETRYABLE_ERRORS = (httplib2.HttpLib2Error, IOError)
@@ -22,23 +29,81 @@ def list_table(list_command):
2229
limit = list_command.limit
2330

2431
if isinstance(list_command, ListCommand):
25-
return table_list_handler.list_all_table(GOOGLE_BIGQUERY_CLIENT, dataset, limit)
32+
return table_list_handler.list_all_table(GOOGLE_BIGQUERY_CLIENT,
33+
dataset,
34+
limit)
2635
elif isinstance(list_command, ListRegexCommand):
2736
return table_list_handler.list_regex_table(GOOGLE_BIGQUERY_CLIENT,
28-
dataset,
29-
list_command.table_name_pattern,
30-
limit)
37+
dataset,
38+
list_command.table_name_pattern,
39+
limit)
3140
elif isinstance(list_command, ListWildcardCommand):
3241
return table_list_handler.list_wildcard_table(GOOGLE_BIGQUERY_CLIENT,
33-
dataset,
34-
list_command.table_prefix,
35-
list_command.start_date,
36-
list_command.end_date)
42+
dataset,
43+
list_command.table_prefix,
44+
list_command.start_date,
45+
list_command.end_date,
46+
limit)
47+
else:
48+
raise ValueError("Unrecognised command type.")
49+
50+
51+
52+
def delete_table(del_command):
53+
dataset = del_command.dataset
54+
55+
if isinstance(del_command, DeleteNamedCommand):
56+
table_delete_handler.delete_table_by_name(
57+
GOOGLE_BIGQUERY_CLIENT, dataset, del_command.table_name)
58+
elif isinstance(del_command, DeleteFileCommand):
59+
table_delete_handler.delete_table_using_file(
60+
GOOGLE_BIGQUERY_CLIENT, dataset, del_command.delete_file)
61+
elif isinstance(del_command, DeleteWildcardCommand):
62+
table_delete_handler.delete_table_with_wildcard(GOOGLE_BIGQUERY_CLIENT,
63+
dataset,
64+
del_command.table_prefix,
65+
del_command.start_date,
66+
del_command.end_date)
67+
elif isinstance(del_command, DeleteRegexCommand):
68+
table_delete_handler.delete_table_with_regex(GOOGLE_BIGQUERY_CLIENT,
69+
dataset,
70+
del_command.table_name_pattern)
3771
else:
3872
raise ValueError("Unrecognised command type.")
3973

74+
def __get_delete_tables(del_command):
75+
dataset = del_command.dataset
4076

77+
if isinstance(del_command, DeleteNamedCommand):
78+
return [del_command.table_name]
4179

80+
elif isinstance(del_command, DeleteFileCommand):
81+
arg_file_path = del_command.delete_file
82+
if os.path.isabs(arg_file_path):
83+
delete_file_path = arg_file_path
84+
else:
85+
working_path = os.getcwd()
86+
delete_file_path = os.path.join(working_path, arg_file_path)
87+
88+
if not os.path.exists(delete_file_path):
89+
raise ValueError("Given file path doesn't exist: %s" % arg_file_path)
90+
91+
with open(delete_file_path) as table_list_file:
92+
return table_list_file.readlines()
93+
94+
elif isinstance(del_command, DeleteWildcardCommand):
95+
return table_list_handler.list_wildcard_table(GOOGLE_BIGQUERY_CLIENT,
96+
dataset,
97+
del_command.table_prefix,
98+
del_command.start_date,
99+
del_command.end_date)
100+
elif isinstance(del_command, DeleteRegexCommand):
101+
return table_list_handler.list_regex_table(GOOGLE_BIGQUERY_CLIENT,
102+
dataset,
103+
del_command.table_name_pattern,
104+
sys.maxint)
105+
else:
106+
raise ValueError("Unrecognised delete command.")
42107

43108
def __get_bigquery_service():
44109
credentials = GoogleCredentials.get_application_default()

‎bquick/bigquery/table_delete_handler.py‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
# -*- coding: utf-8 -*-
2+
3+
import sys
4+
import os
5+
import copy
6+
from collections import namedtuple
7+
from multiprocessing import Pool
8+
from bquick.config_parser import get_config
9+
from bquick.bigquery import table_list_handler
10+
11+
BQ_CONFIG = get_config()
12+
13+
DeleteProcessParam = namedtuple("DeleteProcessParam", "bq_client \
14+
dataset \
15+
table_name")
16+
17+
def delete_table_by_name(bq_client, dataset, table_name, ignore_confirm=False):
18+
"""Delete the table with the given name.
19+
"""
20+
__delete_table_list(bq_client, dataset, [table_name], ignore_confirm)
21+
22+
def delete_table_using_file(bq_client, dataset, file_path,
23+
ignore_confirm=False):
24+
"""Delete all the tables in the given file.
25+
"""
26+
arg_file_path = file_path
27+
if os.path.isabs(arg_file_path):
28+
delete_file_path = arg_file_path
29+
else:
30+
working_path = os.getcwd()
31+
delete_file_path = os.path.join(working_path, arg_file_path)
32+
33+
if not os.path.exists(delete_file_path):
34+
raise ValueError("Given file path doesn't exist: %s" % arg_file_path)
35+
36+
with open(delete_file_path) as table_list_file:
37+
table_list = [line.rstrip() for line in table_list_file.readlines()]
38+
__delete_table_list(bq_client, dataset, table_list, ignore_confirm)
39+
40+
def delete_table_with_wildcard(bq_client,
41+
dataset,
42+
table_prefix,
43+
start_date,
44+
end_date,
45+
ignore_confirm=False):
46+
"""Delete all the tables matching the wildcard table prefix.
47+
"""
48+
table_list = table_list_handler.list_wildcard_table(bq_client,
49+
dataset,
50+
table_prefix,
51+
start_date,
52+
end_date,
53+
sys.maxint)
54+
__delete_table_list(bq_client, dataset, table_list, ignore_confirm)
55+
56+
def delete_table_with_regex(bq_client, dataset, table_name_pattern,
57+
ignore_confirm=False):
58+
"""Delete all the tables that match given regex pattern.
59+
"""
60+
table_list = table_list_handler.list_regex_table(bq_client,
61+
dataset,
62+
table_name_pattern,
63+
sys.maxint)
64+
__delete_table_list(bq_client, dataset, table_list, ignore_confirm)
65+
66+
67+
def __delete_table_list(bq_client, dataset, table_name_list, ignore_confirm):
68+
"""Delete all the tables in the list
69+
"""
70+
if not ignore_confirm:
71+
if not __confirm_delete(dataset, table_name_list):
72+
return
73+
74+
# TODO: make parallel happen
75+
# delete_process_pool = Pool(processes=2)
76+
# delete_process_pool.map(
77+
# __delete_table_process,
78+
# [DeleteProcessParam(bq_client=copy.deepcopy(bq_client),
79+
# dataset=dataset,
80+
# table_name=table_name) \
81+
# for table_name in table_name_list])
82+
# delete_process_pool.close()
83+
# delete_process_pool.join()
84+
for table_name in table_name_list:
85+
param = DeleteProcessParam(bq_client=bq_client,
86+
dataset=dataset,
87+
table_name=table_name)
88+
__delete_table_process(param)
89+
90+
def __confirm_delete(dataset, table_name_list):
91+
"""This function will block process and ask for confirmation.
92+
All the table names will be shown.
93+
"""
94+
proceed_choices = ['yes', 'y']
95+
abort_choices = ['no', 'n']
96+
97+
for table_name in table_name_list:
98+
print table_name
99+
100+
print "Delete all these tables from dataset [%s]? [y or n]" % dataset
101+
102+
while True:
103+
choice = raw_input().lower()
104+
if choice in proceed_choices:
105+
return True
106+
elif choice in abort_choices:
107+
return False
108+
else:
109+
print "Please enter [y or n]"
110+
111+
def __delete_table_process(param):
112+
"""Delete table with compressed param:
113+
Args:
114+
- param: A dict consists of:
115+
- bq_client: a bigquery service object.
116+
- dataset: the dataset to operate on.
117+
- table_name: of which table to be deleted.
118+
"""
119+
bq_client = param.bq_client
120+
dataset = param.dataset
121+
table_name = param.table_name
122+
123+
__delete_table(bq_client, dataset, table_name)
124+
125+
126+
127+
def __delete_table(bq_client, dataset, table_name):
128+
"""Delete the table with given name.
129+
"""
130+
try:
131+
bq_client.tables() \
132+
.delete(projectId=BQ_CONFIG.project,
133+
datasetId=dataset,
134+
tableId=table_name) \
135+
.execute()
136+
except Exception as e:
137+
# TODO: Logging
138+
raise

0 commit comments

Comments
 (0)