Commit 7c32ec7d authored by Gaute Hope's avatar Gaute Hope

remote: push tag changes

parent cb07fa35
......@@ -3,4 +3,13 @@
Checks and fetches e-mail from gmail accounts to a maildir, can synchronize
maildir flags and notmuch tags back to gmail.
# pull
will pull down all remote changes since last time, overwriting any local tag
changes of the affected messages.
# push
will push up all changes since last push, overwriting any remote changes since
the previous pull of the affected messages.
......@@ -8,6 +8,7 @@ import os, sys
import argparse
from oauth2client import tools
import googleapiclient
import notmuch
from tqdm import tqdm, tqdm_gui
......@@ -42,7 +43,7 @@ class Gmailieer:
help = 'Force action (auth)')
parser.add_argument ('--limit', type = int, default = None,
help = 'Maximum number of messages to synchronize')
help = 'Maximum number of messages to synchronize (soft limit, gmail may return more)')
args = parser.parse_args (sys.argv[1:])
self.args = args
......@@ -69,12 +70,56 @@ class Gmailieer:
self.remote.authorize (self.force)
elif self.action == 'push':
raise NotImplmentedError ()
self.push ()
elif self.action == 'init':
self.local.initialize_repository ()
def push (self):
self.remote.get_labels ()
self.local.load_repository ()
# check if remote repository has changed
try:
cur_hist = self.remote.get_current_history_id (self.local.state.last_historyId)
except googleapiclient.errors.HttpError:
print ("historyId is too old, full pull required.")
return
if cur_hist > self.local.state.last_historyId or cur_hist == -1:
print ("push: remote has changed, changes may be overwritten (%d > %d)" % (cur_hist, self.local.state.last_historyId))
if not self.force:
return
# loading local changes
self.local.notmuch = notmuch.Database ()
(rev, uuid) = self.local.notmuch.get_revision ()
if rev == self.local.state.lastmod:
print ("everything is up-to-date.")
return
qry = "path:%s/** and lastmod:%d..%d" % (self.local.nm_relative, self.local.state.lastmod, rev)
# print ("collecting changes..: %s" % qry)
query = notmuch.Query (self.local.notmuch, qry)
total = query.count_messages () # might be destructive here as well
query = notmuch.Query (self.local.notmuch, qry)
messages = list(query.search_messages ())
if self.limit is not None and len(messages) > self.limit:
messages = messages[:self.limit]
# push changes
bar = tqdm (leave = True, total = len(messages), desc = 'pushing changes')
for m in messages:
self.remote.update (m)
bar.update (1)
bar.close ()
if not self.dry_run:
self.local.state.set_lastmod (rev)
def pull (self):
if self.list_labels:
......@@ -133,7 +178,8 @@ class Gmailieer:
# get historyId
mm = self.remote.get_message (message_ids[0])
last_id = int(mm['historyId'])
self.local.state.set_last_history_id (last_id)
if not self.dry_run:
self.local.state.set_last_history_id (last_id)
# get content for new messages
updated = self.get_content (message_ids)
......@@ -177,7 +223,8 @@ class Gmailieer:
# get historyId
mm = self.remote.get_message (message_ids[0])
last_id = int(mm['historyId'])
self.local.state.set_last_history_id (last_id)
if not self.dry_run:
self.local.state.set_last_history_id (last_id)
# get content for new messages
updated = self.get_content (message_ids)
......@@ -203,7 +250,10 @@ class Gmailieer:
bar.update (1)
self.local.update_tags (m)
self.local.notmuch = notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE)
self.remote.get_messages (msgids, _got_msg, 'minimal')
self.local.notmuch.close ()
self.local.notmuch = None
bar.close ()
......
......@@ -2,8 +2,11 @@ import os
import json
import base64
import notmuch
class Local:
wd = None
notmuch = None
translate_labels = {
......@@ -18,8 +21,12 @@ class Local:
'CHAT' : 'chat'
}
labels_translate = { v: k for k, v in translate_labels.items () }
replace_slash_with_dot = True
ignore_labels = set (['attachment', 'encrypted', 'signed', 'new'])
class RepositoryException (Exception):
pass
......@@ -30,6 +37,9 @@ class Local:
# sync.
last_historyId = 0
# this is the last modification id of the notmuch db when the previous push was completed.
lastmod = 0
def __init__ (self, state_f):
self.state_f = state_f
......@@ -40,11 +50,13 @@ class Local:
self.json = {}
self.last_historyId = self.json.get ('last_historyId', 0)
self.lastmod = self.json.get ('lastmod', 0)
def write (self):
self.json = {}
self.json['last_historyId'] = self.last_historyId
self.json['lastmod'] = self.lastmod
with open (self.state_f, 'w') as fd:
json.dump (self.json, fd)
......@@ -53,6 +65,10 @@ class Local:
self.last_historyId = hid
self.write ()
def set_lastmod (self, m):
self.lastmod = m
self.write ()
def __init__ (self, g):
self.gmailieer = g
self.wd = os.getcwd ()
......@@ -85,7 +101,25 @@ class Local:
self.files.extend (fnames)
break
self.mids = [f.split(':')[0] for f in self.files]
self.mids = {}
for f in self.files:
m = f.split (':')[0]
self.mids[m] = f
## Check if we are in the notmuch db
self.notmuch = notmuch.Database ()
try:
self.nm_dir = self.notmuch.get_directory (os.path.abspath(os.path.join (self.md, '..'))).path
self.nm_relative = self.nm_dir[len(self.notmuch.get_path ())+1:]
self.has_notmuch = True
except notmuch.errors.FileError:
print ("error: local mail repository not in notmuch db")
self.has_notmuch = False
self.notmuch.close ()
self.notmuch = None
def initialize_repository (self):
"""
......@@ -141,7 +175,7 @@ class Local:
bname = self.__make_maildir_name__(mid, labels)
self.files.append (bname)
self.mids.append (mid)
self.mids[mid] = bname
p = os.path.join (self.md, bname)
if os.path.exists (p):
......@@ -151,12 +185,12 @@ class Local:
with open (p, 'wb') as fd:
fd.write (msg_str)
# add to notmuch
self.update_tags (m)
self.update_tags (m, p)
def update_tags (self, m):
def update_tags (self, m, fname = None):
# make sure notmuch tags reflect gmail labels
mid = m['id']
labels = m['labelIds']
# translate labels. Remote.get_labels () must have been called first
......@@ -165,8 +199,7 @@ class Local:
labels = set(labels)
# remove ignored labels
ignore_labels = set (self.gmailieer.remote.ignore_labels)
labels = list(labels - ignore_labels)
labels = list(labels - self.gmailieer.remote.ignore_labels)
# translate to notmuch tags
labels = [self.translate_labels.get (l, l) for l in labels]
......@@ -175,5 +208,45 @@ class Local:
if self.replace_slash_with_dot:
labels = [l.replace ('/', '.') for l in labels]
# print (labels)
if fname is None:
# this file probably already exists and just needs it tags updated,
# let's try to find its name in the mid to fname table.
fname = self.mids[mid]
fname = os.path.join (self.md, fname)
if self.notmuch is None:
db = notmuch.Database(mode = notmuch.Database.MODE.READ_WRITE)
else:
db = self.notmuch
nmsg = db.find_message_by_filename (fname)
if nmsg is None:
if self.dry_run:
print ("(dry-run) adding message: %s: %s, with tags: %s" % (mid, fname, str(labels)))
else:
(nmsg, stat) = db.add_message (fname, True)
nmsg.freeze ()
# adding initial tags
for t in labels:
nmsg.add_tag (t, True)
nmsg.thaw ()
else:
# message is already in db, set local tags to match remote tags
otags = nmsg.get_tags ()
if set(otags) != set (labels):
if not self.dry_run:
nmsg.freeze ()
nmsg.remove_all_tags ()
for t in labels:
nmsg.add_tag (t, False)
nmsg.thaw ()
nmsg.tags_to_maildir_flags ()
else:
print ("(dry-run) changing tags on message: %s from: %s to: %s" % (mid, str(otags), str(labels)))
if self.notmuch is None:
db.close ()
......@@ -6,7 +6,7 @@ from oauth2client import tools
from oauth2client.file import Storage
class Remote:
SCOPES = 'https://www.googleapis.com/auth/gmail.readonly https://www.googleapis.com/auth/gmail.labels'
SCOPES = 'https://www.googleapis.com/auth/gmail.readonly https://www.googleapis.com/auth/gmail.labels https://www.googleapis.com/auth/gmail.modify'
APPLICATION_NAME = 'Gmailieer'
CLIENT_SECRET_FILE = None
authorized = False
......@@ -30,12 +30,12 @@ class Remote:
# these cannot be changed manually
read_only_labels = [ 'SENT', 'DRAFT' ]
ignore_labels = [ 'CATEGORY_PERSONAL',
'CATEGORY_SOCIAL',
'CATEGORY_PROMOTIONS',
'CATEGORY_UPDATES',
'CATEGORY_FORUMS'
]
ignore_labels = set([ 'CATEGORY_PERSONAL',
'CATEGORY_SOCIAL',
'CATEGORY_PROMOTIONS',
'CATEGORY_UPDATES',
'CATEGORY_FORUMS'
])
class BatchException (Exception):
pass
......@@ -44,6 +44,7 @@ class Remote:
self.gmailieer = g
self.CLIENT_SECRET_FILE = g.credentials_file
self.account = g.account
self.dry_run = g.dry_run
def __require_auth__ (func):
def func_wrap (self, *args, **kwargs):
......@@ -63,6 +64,17 @@ class Remote:
return self.labels
@__require_auth__
def get_current_history_id (self, start):
"""
Get the current history id of the mailbox
"""
results = self.service.users ().history ().list (userId = self.account, startHistoryId = start).execute ()
if 'history' in results:
return int(results['historyId'])
else:
return start
@__require_auth__
def get_messages_since (self, start):
"""
......@@ -141,7 +153,7 @@ class Remote:
print ("reducing batch request size to: %d" % max_req)
else:
raise Remote.BatchException ("cannot reduce request any further")
@__require_auth__
def get_message (self, mid, format = 'minimal'):
......@@ -193,3 +205,72 @@ class Remote:
print('Storing credentials to ' + credential_path)
return credentials
@__require_auth__
def update (self, m):
"""
Gets a message and checks which labels it should add and which to delete.
"""
# get gmail id
fname = m.get_filename ()
mid = os.path.basename (fname).split (':')[0]
# first get message and figure out what labels it has now
r = self.get_message (mid)
labels = r['labelIds']
labels = [self.labels[l] for l in labels]
# remove ignored labels
labels = set (labels)
labels = labels - self.ignore_labels
# translate to notmuch tags
labels = [self.gmailieer.local.translate_labels.get (l, l) for l in labels]
# this is my weirdness
if self.gmailieer.local.replace_slash_with_dot:
labels = [l.replace ('/', '.') for l in labels]
labels = set(labels)
# current tags
tags = set(m.get_tags ())
# remove special notmuch tags
tags = tags - self.gmailieer.local.ignore_labels
add = list(tags - labels)
rem = list(labels - tags)
# translate back to gmail labels
add = [self.gmailieer.local.labels_translate.get (k, k) for k in add]
rem = [self.gmailieer.local.labels_translate.get (k, k) for k in rem]
if self.gmailieer.local.replace_slash_with_dot:
add = [a.replace ('.', '/') for a in add]
rem = [r.replace ('.', '/') for r in rem]
if len(add) > 0 or len(rem) > 0:
if self.dry_run:
print ("(dry-run) mid: %s: add: %s, remove: %s" % (mid, str(add), str(rem)))
else:
self.__push_tags__ (mid, add, rem)
@__require_auth__
def __push_tags__ (self, mid, add, rem):
"""
Push message changes (these are currently not batched)"
"""
add = [self.labels[a] for a in add]
rem = [self.labels[r] for r in rem]
body = { 'addLabelIds' : add,
'removeLabelIds' : rem }
result = self.service.users ().messages ().modify (userId = self.account,
id = mid, body = body).execute ()
return result
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment