Change backup to use GAE's new __key__ paging technique
Recently Google App Engine learned how to access the entity primary
key as part of the query's where clause, permitting range scans
by key. This allows us to efficiently page the results and get
through all entities without making updates to them.
Unfortunately the change means its harder to resume if we abort,
as we don't have a way to pick up where we left off from.
Signed-off-by: Shawn O. Pearce <sop@google.com>
diff --git a/proto/backup.proto b/proto/backup.proto
index 2b13c61..44cfe12 100644
--- a/proto/backup.proto
+++ b/proto/backup.proto
@@ -17,7 +17,7 @@
message NextChunkRequest {
required string kind = 1;
- required int32 last_backed_up = 2;
+ required string last_key = 2;
}
message EntityXml {
@@ -34,21 +34,9 @@
message EntityAck {
required string key = 1;
- required int32 last_backed_up = 2;
-}
-
-message AckChunkRequest {
- repeated EntityAck entity = 1;
- required int32 last_backed_up = 2;
-}
-
-message AckChunkResponse {
}
service BackupService {
rpc NextChunk (NextChunkRequest)
returns (NextChunkResponse);
-
- rpc AckChunk (AckChunkRequest)
- returns (AckChunkResponse);
}
diff --git a/webapp/backup_gae.py b/webapp/backup_gae.py
index 0b510cc..116672b 100755
--- a/webapp/backup_gae.py
+++ b/webapp/backup_gae.py
@@ -78,9 +78,6 @@
help="Do not save authentication cookies to local disk.")
group = parser.add_option_group("Backup database options")
-group.add_option('-v', action='store', dest='last_backed_up',
- default=0,
- help='Value for last_backed_up query')
group.add_option("-d", action="store", dest="dbname",
metavar="DBNAME",
help="PostgreSQL database name")
@@ -476,25 +473,23 @@
db.cursor().execute("set client_encoding to unicode")
store = LocalStore(db)
- last_backed_up = int(options.last_backed_up)
- print 'BEGIN BACKUP %d' % last_backed_up
+ print 'BEGIN BACKUP'
for kind_name in KINDS:
sys.stdout.write('\n')
cnt = 0
+ last_key = ''
while True:
sys.stdout.write('\r%-18s ... ' % kind_name)
r = NextChunkRequest()
r.kind = kind_name
- r.last_backed_up = last_backed_up
+ r.last_key = last_key
r = backup.NextChunk(r)
if not r.entity:
break
- ack = AckChunkRequest()
- ack.last_backed_up = last_backed_up + 1
for entity in r.entity:
cnt += 1
sys.stdout.write('\r%-18s ... %5d ' % (kind_name, cnt))
@@ -506,14 +501,11 @@
'</root>'
% entity.xml))
getattr(store, 'save_%s' % kind_name)(entity, o)
- a = ack.entity.add()
- a.key = entity.key
- a.last_backed_up = entity.last_backed_up
+ last_key = entity.key
db.commit()
- backup.AckChunk(ack)
sys.stdout.write('\n')
- print 'BACKUP %d DONE' % last_backed_up
+ print 'BACKUP DONE'
db.commit()
db.close()
diff --git a/webapp/codereview/backup_service.py b/webapp/codereview/backup_service.py
index b4f8eda..f344f5a 100644
--- a/webapp/codereview/backup_service.py
+++ b/webapp/codereview/backup_service.py
@@ -57,10 +57,15 @@
done(rsp)
return
+ if req.last_key:
+ q = models.gql(cls,
+ 'WHERE __key__ > :1 ORDER BY __key__',
+ db.Key(req.last_key))
+ else:
+ q = models.gql(cls, 'ORDER BY __key__')
+
sz = 20
- for o in models.gql(cls,
- 'WHERE last_backed_up <= :1',
- req.last_backed_up).fetch(sz):
+ for o in q.fetch(sz):
e = rsp.entity.add()
if o.key().id() is not None:
e.key_id = o.key().id()
@@ -74,19 +79,3 @@
e.xml = o.to_xml().encode('utf_8')
done(rsp)
-
- @automatic_retry
- def AckChunk(self, rpc_controller, req, done):
- if not self.http_request.user_is_admin:
- raise GetLostError()
-
- for i in req.entity:
- def t():
- o = db.get(i.key)
- if o.last_backed_up == i.last_backed_up:
- o.last_backed_up = req.last_backed_up
- o.put()
- db.run_in_transaction(t)
-
- rsp = AckChunkResponse()
- done(rsp)
