Commit 3e24c71e authored by Sriram Karra's avatar Sriram Karra

Rewrite the sync logic for Outlook -> Google

Rewrite in a generic folder -> folder fashion, and tested outlook to
google.
parent 8c25684a
##
## Created : Tue Mar 13 14:26:01 IST 2012
## Last Modified : Fri Mar 30 16:47:49 IST 2012
## Last Modified : Mon Apr 02 14:08:49 IST 2012
##
## Copyright (C) 2012 Sriram Karra <karra.etc@gmail.com>
##
......@@ -16,6 +16,8 @@ from abc import ABCMeta, abstractmethod
from pimdb import PIMDB
from item import Item
import copy, logging
class Contact(Item):
__metaclass__ = ABCMeta
......@@ -28,26 +30,26 @@ class Contact(Item):
Item.__init__(self, folder)
self.props.update({'firstname' : None, 'company' : None,
'lastname' : None, 'postal' : None,
'name' : None, 'notes' : [],
'suffix' : None, 'phone_home' : [],
'title' : None, 'phone_work' : [],
'gender' : None, 'phone_mob' : [],
'nickname' : None, 'phone_other' : [],
'birthday' : None, 'phone_prim' : None,
'anniv' : None, 'fax_home' : [],
'web_home' : [], 'fax_work' : [],
'web_work' : [], 'fax_prim' : None,
'web_prim' : None, 'email_home' : [],
'dept' : None, 'email_work' : [],
'fileas' : None, 'email_other' : [],
'prefix' : None, 'email_prim' : None,
'im' : {}, 'im_prim' : None,
})
if con:
self.init_props_from_con(con)
else:
self.props.update({'firstname' : None, 'company' : None,
'lastname' : None, 'postal' : None,
'name' : None, 'notes' : [],
'suffix' : None, 'phone_home' : [],
'title' : None, 'phone_work' : [],
'gender' : None, 'phone_mob' : [],
'nickname' : None, 'phone_other' : [],
'birthday' : None, 'phone_prim' : None,
'anniv' : None, 'fax_home' : [],
'web_home' : [], 'fax_work' : [],
'web_work' : [], 'fax_prim' : None,
'web_prim' : None, 'email_home' : [],
'dept' : None, 'email_work' : [],
'fileas' : None, 'email_other' : [],
'prefix' : None, 'email_prim' : None,
'im' : {}, 'im_prim' : None,
})
##
## Now onto the non-abstract methods. We do not want to use method
......@@ -56,28 +58,19 @@ class Contact(Item):
## plain english like so.
##
def init_props_from_con (self, con, excl_itemid=True):
def init_props_from_con (self, con):
"""Make a deepcopy of all the item properties from con into the props
dictionary, utilizing the appropriate get_ and set_ routines.
By default the itemid field is excluded from the copy as the most
common usecase is to make a copy of the contact fields in one
database format into another database format - i.e. Create an
OLContact object from the fields of a GCContact object. In such an
instance the itemid field in the destination should really be
generated from a store in the database on creation, or through some
other such means."""
"""
prop_names = con.get_prop_names()
if excl_itemid:
prop_names.remove('itemid')
for prop in prop_names:
get_method = 'get_%s' % prop
set_method = 'set_%s' % prop
val = copy.deepcopy(getattr(con, get_method)())
print 'setting value (', val, ') using method ', set_method
# logging.debug('setting value (%s) using method: %s',
# val, set_method)
getattr(self, set_method)(val)
def get_firstname (self):
......@@ -85,14 +78,14 @@ class Contact(Item):
def set_firstname (self, val):
self._set_prop('firstname', val)
self.update_fullname()
# self.update_fullname()
def get_lastname (self):
return self._get_prop('lastname')
def set_lastname (self, val):
self._set_prop('lastname', val)
self.update_fullname()
# self.update_fullname()
def update_fullname (self):
pr = self.get_prefix()
......@@ -322,7 +315,7 @@ class Contact(Item):
return self._get_prop('im_prim')
def set_im_prim (self, val):
return self_.set_prop('im_prim', val)
return self._set_prop('im_prim', val)
def get_im (self, which=None):
all_ims = self._get_prop('im')
......
##
## Created : Tue Mar 13 14:26:01 IST 2012
## Last Modified : Sat Mar 31 01:10:18 IST 2012
## Last Modified : Mon Apr 02 20:07:40 IST 2012
##
## Copyright (C) 2012 Sriram Karra <karra.etc@gmail.com>
##
......@@ -25,6 +25,18 @@ class GCContact(Contact):
def __init__ (self, folder, con=None, gce=None):
Contact.__init__(self, folder, con)
## Sometimes we might be creating a contact object from Outlook or
## other entry which might have the google contact ID in its sync tags
## field. if that is present, we should use it to initialize the
## itemid field for the current object
try:
## FIXME: fix the hard coded stuff below.
itemid = con.get_sync_tags('asynk:gc:id')
self.set_itemid(itemid)
except KeyError, e:
logging.debug("Hm, nothing found; move on.")
self.set_gce(gce)
if gce:
self.init_props_from_gce(gce)
......@@ -57,9 +69,9 @@ class GCContact(Contact):
## Now onto the non-abstract methods.
##
def get_gce (self):
def get_gce (self, refresh=False):
gce = self._get_att('gce')
if gce:
if gce and (not refresh):
return gce
return self.init_gce_from_props()
......@@ -78,6 +90,7 @@ class GCContact(Contact):
self._snarf_dates_from_gce(gce)
self._snarf_websites_from_gce(gce)
self._snarf_ims_from_gce(gce)
self._snarf_sync_tags_from_gce(gce)
self._snarf_custom_props_from_gce(gce)
......@@ -95,6 +108,7 @@ class GCContact(Contact):
self._add_dates_to_gce(gce)
self._add_websites_to_gce(gce)
self._add_ims_to_gce(gce)
self._add_sync_tags_to_gce(gce)
self._add_custom_props_to_gce(gce)
......@@ -115,6 +129,7 @@ class GCContact(Contact):
def _snarf_itemid_from_gce (self, ce):
if ce.id:
logging.debug('set itemid for google entry')
self.set_itemid(ce.id.text)
def _snarf_names_gender_from_gce (self, ce):
......@@ -242,7 +257,16 @@ class GCContact(Contact):
if im.prim:
self.set_im_prim(im.address)
def _snarf_custom_props_from_gce (self, ce):
def _snarf_sync_tags_from_gce (self, ce):
logging.error("_snarf_sync_tags(): Not Implemented Yet")
if ce.user_defined_field:
keyprefix = (self.get_config().get_label_prefix() +
self.get_config().get_label_separator())
self.set_sync_tags(get_udp_by_key_prefix(ce.user_defined_field,
keyprefix))
print '===== sync_tags: ', self.get_sync_tags()
def _snarf_custom_props_from_gce (self, ce):
logging.error("_snarf_custom_props(): Not Implemented Yet")
def _is_valid_phone_number (self, phone, type, name):
......@@ -260,7 +284,7 @@ class GCContact(Contact):
def _add_itemid_to_gce (self, gce):
itemid = self.get_itemid()
if itemid:
gce.id = atom.data.Id(text=gcid)
gce.id = atom.data.Id(text=itemid)
def _add_names_gender_to_gce (self, gce):
"""Populate the Name fields in gce, which is a Google ContactEntry
......@@ -278,7 +302,6 @@ class GCContact(Contact):
n.family_name = gdata.data.FamilyName(text=text)
text = self.get_name()
print 'name: ', text
if text:
n.full_name = gdata.data.FullName(text=text)
......@@ -337,18 +360,24 @@ class GCContact(Contact):
email_prim = self.get_email_prim()
for email in self.get_email_home():
if not email:
continue
prim = 'true' if email == email_prim else 'false'
em = gdata.data.Email(address=email, primary=prim,
rel=gdata.data.HOME_REL)
gce.email.append(em)
for email in self.get_email_work():
if not email:
continue
prim = 'true' if email == email_prim else 'false'
em = gdata.data.Email(address=email, primary=prim,
rel=gdata.data.WORK_REL)
gce.email.append(em)
for email in self.get_email_other():
if not email:
continue
prim = 'true' if email == email_prim else 'false'
em = gdata.data.Email(address=email, primary=prim,
rel=gdata.data.OTHER_REL)
......@@ -395,24 +424,32 @@ class GCContact(Contact):
ph_prim = self.get_phone_prim()
for ph in self.get_phone_home():
if not ph:
continue
prim = 'true' if ph == ph_prim else 'false'
phone = gdata.data.PhoneNumber(text=ph, primary=prim,
rel=gdata.data.HOME_REL)
gce.phone_number.append(phone)
for ph in self.get_phone_work():
if not ph:
continue
prim = 'true' if ph == ph_prim else 'false'
phone = gdata.data.PhoneNumber(text=ph, primary=prim,
rel=gdata.data.WORK_REL)
gce.phone_number.append(phone)
for ph in self.get_phone_other():
if not ph:
continue
prim = 'true' if ph == ph_prim else 'false'
phone = gdata.data.PhoneNumber(text=ph, primary=prim,
rel=gdata.data.OTHER_REL)
gce.phone_number.append(phone)
for ph in self.get_phone_mob():
if not ph:
continue
prim = 'true' if ph == ph_prim else 'false'
phone = gdata.data.PhoneNumber(text=ph, primary=prim,
rel=gdata.data.MOBILE_REL)
......@@ -421,12 +458,16 @@ class GCContact(Contact):
fax_prim = self.get_fax_prim()
for fa in self.get_fax_home():
if not fa:
continue
prim = 'true' if fa == fax_prim else 'false'
fax = gdata.data.PhoneNumber(text=ph, primary=prim,
rel=gdata.data.HOME_FAX_REL)
gce.phone_number.append(fax)
for fa in self.get_fax_work():
if not fa:
continue
prim = 'true' if fa == fax_prim else 'false'
fax = gdata.data.PhoneNumber(text=ph, primary=prim,
rel=gdata.data.WORK_FAX_REL)
......@@ -453,12 +494,16 @@ class GCContact(Contact):
web_prim = self.get_web_prim()
for web in self.get_web_home():
if not web:
continue
prim = 'true' if web == web_prim else 'false'
home = gdata.contacts.data.Website(href=web, primary=prim,
rel='home-page')
gce.website.append(home)
for web in self.get_web_work():
if not web:
continue
prim = 'true' if web == web_prim else 'false'
work = gdata.contacts.data.Website(href=web, primary=prim,
rel='work')
......@@ -466,11 +511,21 @@ class GCContact(Contact):
def _add_ims_to_gce (self, gce):
im_prim = self.get_im_prim()
for label, addr in self.get_ims().iteritems():
for label, addr in self.get_im().iteritems():
prim = 'true' if im_prim == addr else 'false'
im = gdata.data.Im(label=label, address=addr, primary=prim)
gce.im.append(im)
def _add_sync_tags_to_gce (self, gce):
## These will be stored as extended properties. Note that if this
## routine keeps appending the sync_tags tot he user_defined_fields,
## with no regard for whether it already exists or not...
for key, val in self.get_sync_tags().iteritems():
ud = gdata.contacts.data.UserDefinedField()
ud.key = key
ud.value = val
gce.user_defined_field.append(ud)
def _add_custom_props_to_gce (self, gce):
## FIXME: This needs to get implemented on priority. This is where all
## the sync tags and stuff will get stored.
......
This diff is collapsed.
##
## Created : Tue Mar 13 14:26:01 IST 2012
## Last Modified : Sat Mar 31 00:29:42 IST 2012
## Last Modified : Mon Apr 02 14:19:40 IST 2012
##
## Copyright (C) 2012 Sriram Karra <karra.etc@gmail.com>
##
......@@ -58,6 +58,20 @@ class Folder:
def __str__ (self):
raise NotImplementedError
@abstractmethod
def get_batch_size (self):
"""When entries are copied during a sync operation, it is frequently
efficient to do them in batch operations - when the underlying
database provider supports it. For e.g. it is a huge performance boost
to use Google Contacts' batch operations. This routine is expected to
return the size of such a batch operation in general. In practcise
what this means is that a sync operation will send this many Contact
entries for processing to the folder / db provider in one go. The
actual batching and handling, will be done by the underlying code
anyway."""
raise NotImplementedError
@abstractmethod
def prep_sync_lists (self, destid, last_sync_stop=None, limit=0):
"""Prepare and return a set of list of new, modified and deleted
......@@ -78,7 +92,15 @@ class Folder:
raise NotImplementedError
@abstractmethod
def insert_new_items (self, items):
def find_item (self, itemid):
"""Return an object of some type derived from Item that is specified
by the given itemid. It is an error for the given itemid to not be
present."""
raise NotImplementedError
@abstractmethod
def batch_create (self, items):
"""Insert new items into the database. entries is a list of objects
derived from pimdb.Item. This routine will ensure relevant fields are
fetched - which will invoke the source db implementation to get the
......@@ -88,6 +110,13 @@ class Folder:
raise NotImplementedError
@abstractmethod
def batch_update (self, items):
"""Update already existing items to new contents. items is a list of
objects derived from pimdb.Item."""
raise NotImplementedError
@abstractmethod
def bulk_clear_sync_flags (self, dbids):
"""destid should be an array of PIMDB ids.
......
This diff is collapsed.
##
## Created : Wed May 18 13:16:17 IST 2011
## Last Modified : Sun Apr 01 16:23:04 IST 2012
## Last Modified : Mon Apr 02 15:49:41 IST 2012
##
## Copyright (C) 2011, 2012 Sriram Karra <karra.etc@gmail.com>
##
......@@ -21,6 +21,7 @@ if __name__ == "__main__":
from abc import ABCMeta, abstractmethod
from folder import Folder
from win32com.mapi import mapi, mapitags, mapiutil
from contact_ol import OLContact
class OLFolder(Folder):
"""An Outlook folder directly corresponds to a MAPI Folder entity. This
......@@ -48,6 +49,9 @@ class OLFolder(Folder):
## Implementation of some abstract methods inherted from Folder
##
def get_batch_size (self):
return 100
def prep_sync_lists (self, destid, sl, synct_sto=None, cnt=0):
"""See the documentation in folder.Folder"""
......@@ -118,7 +122,21 @@ class OLFolder(Folder):
return (sl.get_news(), sl.get_mods(), sl.get_dels())
def insert_new_items (self, items):
def find_item (self, itemid):
eid = base64.b64decode(itemid)
print 'itemid : ', itemid
print 'eid : ', eid
olc = OLContact(self, eid=eid)
return olc
def batch_create (self, items):
"""See the documentation in folder.Folder"""
raise NotImplementedError
def batch_update (self, items):
"""See the documentation in folder.Folder"""
raise NotImplementedError
......@@ -224,7 +242,7 @@ class OLFolder(Folder):
i += 1
if mapitags.PROP_TYPE(gid_tag) != mapitags.PT_ERROR:
entry = store.OpenEntry(entryid, None, MOD_FLAG)
entry = store.OpenEntry(entryid, None, mapi.MAPI_BEST_ACCESS)
hr, ps = entry.DeleteProps([gid_tag])
entry.SaveChanges(mapi.KEEP_OPEN_READWRITE)
......@@ -463,7 +481,7 @@ class PropTags:
gid = self.config.get_olsync_gid(dbid)
prop_name = [(self.config.get_olsync_guid(), gid)]
prop_type = mapitags.PT_UNICODE
prop_ids = self.def_cf.GetIDsFromNames(prop_name, 0)
prop_ids = self.def_cf.GetIDsFromNames(prop_name, mapi.MAPI_CREATE)
return (prop_type | prop_ids[0])
......
##
## Created : Thu Jul 07 14:47:54 IST 2011
## Last Modified : Mon Mar 26 15:09:37 IST 2012
## Last Modified : Mon Apr 02 08:45:11 IST 2012
##
## Copyright (C) 2011, 2012 by Sriram Karra <karra.etc@gmail.com>
##
......@@ -17,22 +17,6 @@ from pimdb import PIMDB, GoutInvalidPropValueError
from folder import Folder
from folder_gc import GCContactsFolder
def sync_status_str (const):
for name, val in globals().iteritems():
if name[:5] == 'SYNC_' and val == const:
return name
return None
SYNC_OK = 200
SYNC_CREATED = 201
SYNC_NOT_MODIFIED = 304
SYNC_BAD_REQUEST = 400
SYNC_UNAUTHORIZED = 401
SYNC_FORBIDDEN = 403
SYNC_CONFLICT = 409
SYNC_INTERNAL_SERVER_ERROR = 500
class GCPIMDB (PIMDB):
"""GC object is a wrapper for a Google Contacts stream API."""
......@@ -216,114 +200,6 @@ class GCPIMDB (PIMDB):
batch_feed, gdata.contacts.client.DEFAULT_BATCH_URL)
class BatchState:
"""This class is used as a temporary store of state related to batch
operations in the Google API. Useful when we are operating in bulk data
on Google"""
def __init__ (self, num, f, op=None):
self.size = 0
self.cnt = 0
self.num = num
self.f = f
self.operation = op
self.cons = {} # can be either Contact or ContactEntry
def get_size (self):
"""Return size of feed in kilobytess."""
self.size = len(str(self.f))/1024.0
return self.size
def incr_cnt (self):
self.cnt += 1
return self.cnt
def get_cnt (self):
return self.cnt
def get_bnum (self):
return self.num
def add_con (self, olid_b64, con):
self.cons[olid_b64] = con
def get_con (self, olid_b64):
return self.cons[olid_b64]
def get_operation (self):
return self.operation
def set_operation (self, op):
self.operation = op
def process_batch_response (self, resp, bstate):
"""resp is the response feed obtained from a batch operation to
google.
bstate contains the stats and other state for all the Contact
objects involved in the batch operation.
This routine will walk through the batch response entries, and
make note in the outlook database for succesful sync, or handle
errors appropriately."""
op = bstate.get_operation()
cons = []
for entry in resp.entry:
bid = entry.batch_id.text if entry.batch_id else None
if not entry.batch_status:
# There is something seriously wrong with this request.
logging.error('Unknown fatal error in response. Full resp: %s',
entry)
continue
code = int(entry.batch_status.code)
reason = entry.batch_status.reason
if code != SYNC_OK and code != SYNC_CREATED:
# FIXME this code path needs to be tested properly
err = sync_status_str(code)
err_str = '' if err is None else ('Code: %s' % err)
err_str = 'Reason: %s. %s' % (reason, err_str)
if op == 'insert' or op == 'update':
logging.error('Upload to Google failed for: %s: %s',
bstate.get_con(bid).name, err_str)
elif op == 'Writeback olid':
logging.error('Could not complete sync for: %s: %s',
bstate.get_con(bid).name, err_str)
else:
## We could just print a more detailed error for all
## cases. Should do some time FIXME.
logging.error('Sync failed for bid %s: %s',
bid, err_str)
else:
if op == 'query':
con = entry
# We could build and return array for all cases, but
# why waste memory...
cons.append(con)
else:
con = bstate.get_con(bid)
gcid = utils.get_link_rel(entry.link, 'edit')
con.update_prop_by_name([(self.config.get_gc_guid(),
self.config.get_gc_id())],
mapitags.PT_UNICODE,
gcid)
t = None
if op == 'insert':
t = 'created'
elif op == 'update':
t = 'updated'
if t:
name = con.name
logging.info('Successfully %s gmail entry for %s',
t, name)
return cons
def main():
config = Config('../app_state.json')
......
This diff is collapsed.
##
## Created : Tue Jul 26 06:54:41 IST 2011
## Last Modified : Thu Mar 22 12:20:47 IST 2012
## Last Modified : Mon Apr 02 14:54:10 IST 2012
##
## Copyright (C) 2011, 2012 by Sriram Karra <karra.etc@gmail.com>
##
## Licensed under the GPL v3
##
import re
## The follow is a super cool implementation of enum equivalent in
## Python. Taken with a lot of gratitude from this post on Stackoverflow:
## http://stackoverflow.com/a/1695250/987738
......@@ -17,6 +19,24 @@ def enum(*sequential, **named):
enums = dict(zip(sequential, range(len(sequential))), **named)
return type('Enum', (), enums)
def get_sync_label_from_dbid (config, dbid):
ret = (config.get_label_prefix() +
config.get_label_separator() +
dbid + config.get_label_separator() + 'id')
return ret
def get_dbid_from_sync_label (config, label):
pre = config.get_label_prefix()
sep = config.get_label_separator()
reg = (pre + sep + '([a-z0-9]+)' + sep + 'id')
res = re.match(reg, label)
if res:
return res.group(1)
else:
return None
def get_link_rel (links, rel):
"""From a Google data entry links array, fetch the link with the
specifirf 'rel' attribute. examples of values for 'rel' could be:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment