Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migrate to notmuch2 module #320

Draft
wants to merge 7 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ jobs:
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
name: Build (Python ${{ matrix.python-version }})
runs-on: ubuntu-20.04
runs-on: ubuntu-21.04
steps:
- uses: actions/checkout@v2
with:
Expand All @@ -24,11 +24,11 @@ jobs:
shell: bash
run: |
sudo apt-get update
sudo apt-get install -y notmuch python3-notmuch python3-venv flake8
sudo apt-get install -y notmuch python3-notmuch2 python3-venv flake8
python3 -m venv env
source ./env/bin/activate
pip install setuptools setuptools_scm pytest dkimpy
ln -s /usr/lib/python3/dist-packages/notmuch ./env/lib/python*/site-packages
ln -s /usr/lib/python3/dist-packages/notmuch2 ./env/lib/python*/site-packages
- name: flake8 lint
run: |
source ./env/bin/activate
Expand Down
31 changes: 14 additions & 17 deletions afew/Database.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import time
import logging

import notmuch
import notmuch2

from afew.NotmuchSettings import notmuch_settings, get_notmuch_new_tags

Expand Down Expand Up @@ -46,19 +46,18 @@ def __exit__(self, exc_type, exc_value, traceback):
"""
self.close()

def open(self, rw=False, retry_for=180, retry_delay=1, create=False):
def open(self, rw=False, retry_for=180, retry_delay=1):
if rw:
if self.handle and self.handle.mode == notmuch.Database.MODE.READ_WRITE:
if self.handle and self.handle.mode == notmuch2.Database.MODE.READ_WRITE:
return self.handle

start_time = time.time()
while True:
try:
self.handle = notmuch.Database(self.db_path,
mode=notmuch.Database.MODE.READ_WRITE,
create=create)
self.handle = notmuch2.Database(self.db_path,
mode=notmuch2.Database.MODE.READ_WRITE)
break
except notmuch.NotmuchError:
except notmuch2.NotmuchError:
time_left = int(retry_for - (time.time() - start_time))

if time_left <= 0:
Expand All @@ -71,7 +70,8 @@ def open(self, rw=False, retry_for=180, retry_delay=1, create=False):
time.sleep(retry_delay)
else:
if not self.handle:
self.handle = notmuch.Database(self.db_path, create=create)
self.handle = notmuch2.Database(self.db_path,
mode=notmuch2.Database.MODE.READ_ONLY)

return self.handle

Expand All @@ -93,7 +93,7 @@ def do_query(self, query):
:rtype: :class:`notmuch.Query`
"""
logging.debug('Executing query %r' % query)
return notmuch.Query(self.open(), query)
return notmuch2.Database.messages(self.open(), query)

def get_messages(self, query, full_thread=False):
"""
Expand All @@ -106,10 +106,10 @@ def get_messages(self, query, full_thread=False):
:returns: an iterator over :class:`notmuch.Message` objects
"""
if not full_thread:
for message in self.do_query(query).search_messages():
for message in self.do_query(query):
yield message
else:
for thread in self.do_query(query).search_threads():
for thread in self.do_query(query):
for message in self.walk_thread(thread):
yield message

Expand Down Expand Up @@ -163,16 +163,13 @@ def add_message(self, path, sync_maildir_flags=False, new_mail_handler=None):
"""
# TODO: it would be nice to update notmuchs directory index here
handle = self.open(rw=True)
if hasattr(notmuch.Database, 'index_file'):
message, status = handle.index_file(path, sync_maildir_flags=sync_maildir_flags)
else:
message, status = handle.add_message(path, sync_maildir_flags=sync_maildir_flags)
message, duplicate = handle.add(path, sync_flags=sync_maildir_flags)

if status != notmuch.STATUS.DUPLICATE_MESSAGE_ID:
if not duplicate:
logging.info('Found new mail in {}'.format(path))

for tag in get_notmuch_new_tags():
message.add_tag(tag)
message.tags.add(tag)

if new_mail_handler:
new_mail_handler(message)
Expand Down
8 changes: 3 additions & 5 deletions afew/MailMover.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@
from datetime import date, datetime, timedelta
from subprocess import check_call, CalledProcessError, DEVNULL

import notmuch

from afew.Database import Database
from afew.utils import get_message_summary

Expand All @@ -21,7 +19,7 @@ class MailMover(Database):

def __init__(self, max_age=0, rename=False, dry_run=False, notmuch_args='', quiet=False):
super().__init__()
self.db = notmuch.Database(self.db_path)
self.db = Database()
self.query = 'folder:"{folder}" AND {subquery}'
if max_age:
days = timedelta(int(max_age))
Expand Down Expand Up @@ -61,11 +59,11 @@ def move(self, maildir, rules):
main_query = self.query.format(
folder=maildir.replace("\"", "\\\""), subquery=query)
logging.debug("query: {}".format(main_query))
messages = notmuch.Query(self.db, main_query).search_messages()
messages = self.db.get_messages(main_query)
for message in messages:
# a single message (identified by Message-ID) can be in several
# places; only touch the one(s) that exists in this maildir
all_message_fnames = message.get_filenames()
all_message_fnames = (str(name) for name in message.filenames())
to_move_fnames = [name for name in all_message_fnames
if maildir in name]
if not to_move_fnames:
Expand Down
10 changes: 5 additions & 5 deletions afew/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import platform
import queue
import threading
import notmuch
import notmuch2
import pyinotify
import ctypes
import contextlib
Expand Down Expand Up @@ -43,19 +43,19 @@ def process_IN_MOVED_TO(self, event):
def new_mail(message):
for filter_ in self.options.enable_filters:
try:
filter_.run('id:"{}"'.format(message.get_message_id()))
filter_.run('id:"{}"'.format(message.messageid))
filter_.commit(self.options.dry_run)
except Exception as e:
logging.warning('Error processing mail with filter {!r}: {}'.format(filter_.message, e))

try:
self.database.add_message(event.pathname,
sync_maildir_flags=True,
sync_flags=True,
new_mail_handler=new_mail)
except notmuch.FileError as e:
except notmuch2.FileError as e:
logging.warning('Error opening mail file: {}'.format(e))
return
except notmuch.FileNotEmailError as e:
except notmuch2.FileNotEmailError as e:
logging.warning('File does not look like an email: {}'.format(e))
return
else:
Expand Down
20 changes: 10 additions & 10 deletions afew/filters/BaseFilter.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,27 +55,27 @@ def run(self, query):
self.handle_message(message)

def handle_message(self, message):
if not self._tag_blacklist.intersection(message.get_tags()):
if not self._tag_blacklist.intersection(message.tags):
self.remove_tags(message, *self._tags_to_remove)
self.add_tags(message, *self._tags_to_add)

def add_tags(self, message, *tags):
if tags:
self.log.debug('Adding tags %s to id:%s' % (', '.join(tags),
message.get_message_id()))
self._add_tags[message.get_message_id()].update(tags)
message.messageid))
self._add_tags[message.messageid].update(tags)

def remove_tags(self, message, *tags):
if tags:
filtered_tags = list(tags)
self.log.debug('Removing tags %s from id:%s' % (', '.join(filtered_tags),
message.get_message_id()))
self._remove_tags[message.get_message_id()].update(filtered_tags)
message.messageid))
self._remove_tags[message.messageid].update(filtered_tags)

def flush_tags(self, message):
self.log.debug('Removing all tags from id:%s' %
message.get_message_id())
self._flush_tags.append(message.get_message_id())
message.messageid)
self._flush_tags.append(message.messageid)

def commit(self, dry_run=True):
dirty_messages = set()
Expand All @@ -93,15 +93,15 @@ def commit(self, dry_run=True):
db = self.database.open(rw=True)

for message_id in dirty_messages:
message = db.find_message(message_id)
message = db.find(message_id)

if message_id in self._flush_tags:
message.remove_all_tags()

for tag in self._add_tags.get(message_id, []):
message.add_tag(tag)
message.tags.add(tag)

for tag in self._remove_tags.get(message_id, []):
message.remove_tag(tag)
message.tags.remove(tag)

self.flush_changes()
10 changes: 7 additions & 3 deletions afew/filters/DKIMValidityFilter.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,18 @@ def __init__(self, database, ok_tag='dkim-ok', fail_tag='dkim-fail'):
self.__module__, self.__class__.__name__))

def handle_message(self, message):
if message.get_header(self.header):
try:
selfhead = message.header(self.header)
except LookupError:
selfhead = ''
if selfhead:
try:
dkim_ok = all(map(verify_dkim, message.get_filenames()))
dkim_ok = all(map(verify_dkim, message.filenames()))
except DKIMVerifyError as verify_error:
self.log.warning(
"Failed to verify DKIM of '%s': %s "
"(marked as 'dkim-fail')",
message.get_message_id(),
message.messageid,
verify_error
)
dkim_ok = False
Expand Down
4 changes: 2 additions & 2 deletions afew/filters/DMARCReportInspectionFilter.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ def __init__(self, # pylint: disable=too-many-arguments
self.__module__, self.__class__.__name__))

def handle_message(self, message):
if not self.dmarc_subject.match(message.get_header('Subject')):
if not self.dmarc_subject.match(message.header('Subject')):
return

auth_results = {'dkim': True, 'spf': True}
Expand All @@ -165,6 +165,6 @@ def handle_message(self, message):
except DMARCInspectionError as inspection_error:
self.log.error(
"Failed to verify DMARC report of '%s': %s (not tagging)",
message.get_message_id(),
message.messageid,
inspection_error
)
10 changes: 7 additions & 3 deletions afew/filters/FolderNameFilter.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ def __init__(self, database, folder_blacklist='', folder_transforms='',
def handle_message(self, message):
# Find all the dirs in the mail directory that this message
# belongs to
maildirs = [re.match(self.__filename_pattern, filename)
for filename in message.get_filenames()]
maildirs = [re.match(self.__filename_pattern, str(filename))
for filename in message.filenames()]
maildirs = filter(None, maildirs)
if maildirs:
# Make the folders relative to mail_root and split them.
Expand All @@ -34,8 +34,12 @@ def handle_message(self, message):
folders = set([folder
for folder_group in folder_groups
for folder in folder_group])
try:
subject = message.header('subject')
except LookupError:
subject = ''
self.log.debug('found folders {} for message {!r}'.format(
folders, message.get_header('subject')))
folders, subject))

# remove blacklisted folders
clean_folders = folders - self.__folder_blacklist
Expand Down
8 changes: 4 additions & 4 deletions afew/filters/HeaderMatchingFilter.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

from afew.filters.BaseFilter import Filter

from notmuch.errors import NullPointerError
from notmuch2._errors import NullPointerError

import re

Expand All @@ -23,14 +23,14 @@ def __init__(self, database, **kwargs):

def handle_message(self, message):
if self.header is not None and self.pattern is not None:
if not self._tag_blacklist.intersection(message.get_tags()):
if not self._tag_blacklist.intersection(message.tags):
try:
value = message.get_header(self.header)
value = message.header(self.header)
match = self.pattern.search(value)
if match:
tagdict = {k: v.lower() for k, v in match.groupdict().items()}
sub = (lambda tag: tag.format(**tagdict))
self.remove_tags(message, *map(sub, self._tags_to_remove))
self.add_tags(message, *map(sub, self._tags_to_add))
except NullPointerError:
except (NullPointerError, LookupError):
pass
2 changes: 1 addition & 1 deletion afew/filters/KillThreadsFilter.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ class KillThreadsFilter(Filter):
query = 'NOT tag:killed'

def handle_message(self, message):
query = self.database.get_messages('thread:"%s" AND tag:killed' % message.get_thread_id())
query = self.database.get_messages('thread:"%s" AND tag:killed' % message.threadid)

if len(list(query)):
self.add_tags(message, 'killed')
2 changes: 1 addition & 1 deletion afew/filters/MeFilter.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,5 @@ def __init__(self, database, me_tag='to-me', tags_blacklist=[]):
self.me_tag = me_tag

def handle_message(self, message):
if not self._tag_blacklist.intersection(message.get_tags()):
if not self._tag_blacklist.intersection(message.tags):
self.add_tags(message, self.me_tag)
4 changes: 2 additions & 2 deletions afew/filters/PropagateTagsByRegexInThreadFilter.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ class PropagateTagsByRegexInThreadFilter(Filter):
"""

def handle_message(self, message):
thread_query = 'thread:"%s"' % (message.get_thread_id(),)
thread_query = 'thread:"%s"' % (message.threadid,)
if self._filter:
query = self.database.get_messages("(%s) AND (%s)" % (thread_query, self._filter))
else:
Expand All @@ -50,7 +50,7 @@ def handle_message(self, message):
messages = list(query)

# flatten tags
tags_in_thread_t = {m.get_tags() for m in messages} # a set of Tags instances
tags_in_thread_t = {m.tags for m in messages} # a set of Tags instances
tags_in_thread = set(_flatten(tags_in_thread_t))

# filter tags
Expand Down
2 changes: 1 addition & 1 deletion afew/filters/PropagateTagsInThreadFilter.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ class PropagateTagsInThreadFilter(Filter):

def handle_message(self, message):
for tag in self._propagate_tags:
tag_query = 'thread:"%s" AND is:"%s"' % (message.get_thread_id(), tag)
tag_query = 'thread:"%s" AND is:"%s"' % (message.threadid, tag)
if self._filter:
query = self.database.get_messages("(%s) AND (%s)" % (tag_query, self._filter))
else:
Expand Down
5 changes: 4 additions & 1 deletion afew/filters/SentMailsFilter.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,10 @@ def handle_message(self, message):
self.add_tags(message, self.sent_tag)
if self.to_transforms:
for header in ('To', 'Cc', 'Bcc'):
email = self.__get_bare_email(message.get_header(header))
try:
email = self.__get_bare_email(message.header(header))
except LookupError:
email = ''
for tag in self.__pick_tags(email):
self.add_tags(message, tag)
else:
Expand Down
Loading
Loading