1 ;;; mh-junk.el --- MH-E interface to anti-spam measures
3 ;; Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
5 ;; Author: Satyaki Das <satyaki@theforce.stanford.edu>,
6 ;; Bill Wohler <wohler@newt.com>
7 ;; Maintainer: Bill Wohler <wohler@newt.com>
8 ;; Keywords: mail, spam
10 ;; This file is part of GNU Emacs.
12 ;; GNU Emacs is free software; you can redistribute it and/or modify
13 ;; it under the terms of the GNU General Public License as published by
14 ;; the Free Software Foundation; either version 2, or (at your option)
17 ;; GNU Emacs is distributed in the hope that it will be useful,
18 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
19 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 ;; GNU General Public License for more details.
22 ;; You should have received a copy of the GNU General Public License
23 ;; along with GNU Emacs; see the file COPYING. If not, write to the
24 ;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
25 ;; Boston, MA 02110-1301, USA.
29 ;; Spam handling in MH-E.
40 (defun mh-junk-blacklist (range)
41 "Blacklist RANGE as spam.
43 This command trains the spam program in use (see the option
44 `mh-junk-program') with the content of RANGE and then handles the
45 message(s) as specified by the option `mh-junk-disposition'.
47 Check the documentation of `mh-interactive-range' to see how RANGE is
48 read in interactive use.
50 For more information about using your particular spam fighting
53 - `mh-spamassassin-blacklist'
54 - `mh-bogofilter-blacklist'
55 - `mh-spamprobe-blacklist'"
56 (interactive (list (mh-interactive-range "Blacklist")))
57 (let ((blacklist-func (nth 1 (assoc mh-junk-choice mh-junk-function-alist))))
58 (unless blacklist-func
59 (error "Customize `mh-junk-program' appropriately"))
60 (let ((dest (cond ((null mh-junk-disposition) nil)
61 ((equal mh-junk-disposition "") "+")
62 ((eq (aref mh-junk-disposition 0) ?+)
64 ((eq (aref mh-junk-disposition 0) ?@)
65 (concat mh-current-folder "/"
66 (substring mh-junk-disposition 1)))
67 (t (concat "+" mh-junk-disposition)))))
68 (mh-iterate-on-range msg range
69 (message "Blacklisting message %d..." msg)
70 (funcall (symbol-function blacklist-func) msg)
71 (message "Blacklisting message %d...done" msg)
72 (if (not (memq msg mh-seen-list))
73 (setq mh-seen-list (cons msg mh-seen-list)))
75 (mh-refile-a-msg nil (intern dest))
76 (mh-delete-a-msg nil)))
80 (defun mh-junk-whitelist (range)
81 "Whitelist RANGE as ham.
83 This command reclassifies the RANGE as ham if it were incorrectly
84 classified as spam (see the option `mh-junk-program'). It then
85 refiles the message into the \"+inbox\" folder.
87 Check the documentation of `mh-interactive-range' to see how
88 RANGE is read in interactive use."
89 (interactive (list (mh-interactive-range "Whitelist")))
90 (let ((whitelist-func (nth 2 (assoc mh-junk-choice mh-junk-function-alist))))
91 (unless whitelist-func
92 (error "Customize `mh-junk-program' appropriately"))
93 (mh-iterate-on-range msg range
94 (message "Whitelisting message %d..." msg)
95 (funcall (symbol-function whitelist-func) msg)
96 (message "Whitelisting message %d...done" msg)
97 (mh-refile-a-msg nil (intern mh-inbox)))
102 ;; Spamassassin Interface
104 (defvar mh-spamassassin-executable (executable-find "spamassassin"))
105 (defvar mh-sa-learn-executable (executable-find "sa-learn"))
108 (defun mh-spamassassin-blacklist (msg)
109 "Blacklist MSG with SpamAssassin.
111 SpamAssassin is one of the more popular spam filtering programs.
112 Get it from your local distribution or from
113 http://spamassassin.org/.
115 To use SpamAssassin, add the following recipes to
118 # Append to $PATH the location of mhparam in some distros.
119 PATH=$PATH:/usr/bin/mh
121 MAILDIR=$HOME/`mhparam Path`
123 # Fight spam with SpamAssassin.
127 # Anything with a spam level of 10 or more is junked immediately.
129 * ^X-Spam-Level: ..........
133 * ^X-Spam-Status: Yes
136 If you don't use \"spamc\", use \"spamassassin -P -a\".
138 Note that one of the recipes above throws away messages with a
139 score greater than or equal to 10. Here's how you can determine a
140 value that works best for you.
142 First, run \"spamassassin -t\" on every mail message in your
143 archive and use Gnumeric to verify that the average plus the
144 standard deviation of good mail is under 5, the SpamAssassin
145 default for \"spam\".
147 Using Gnumeric, sort the messages by score and view the messages
148 with the highest score. Determine the score which encompasses all
149 of your interesting messages and add a couple of points to be
150 conservative. Add that many dots to the \"X-Spam-Level:\" header
151 field above to send messages with that score down the drain.
153 In the example above, messages with a score of 5-9 are set aside
154 in the \"+spam\" folder for later review. The major weakness of
155 rules-based filters is a plethora of false positives so it is
158 If SpamAssassin classifies a message incorrectly, or is unsure,
159 you can use the MH-E commands \\[mh-junk-blacklist] and
160 \\[mh-junk-whitelist].
162 The command \\[mh-junk-blacklist] adds a \"blacklist_from\" entry
163 to \"~/spamassassin/user_prefs\", deletes the message, and sends
164 the message to the Razor, so that others might not see this spam.
165 If the \"sa-learn\" command is available, the message is also
166 recategorized as spam.
168 The command \\[mh-junk-whitelist] adds a \"whitelist_from\" rule
169 to the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\"
170 command is available, the message is also recategorized as ham.
172 Over time, you'll observe that the same host or domain occurs
173 repeatedly in the \"blacklist_from\" entries, so you might think
174 that you could avoid future spam by blacklisting all mail from a
175 particular domain. The utility function
176 `mh-spamassassin-identify-spammers' helps you do precisely that.
177 This function displays a frequency count of the hosts and domains
178 in the \"blacklist_from\" entries from the last blank line in
179 \"~/.spamassassin/user_prefs\" to the end of the file. This
180 information can be used so that you can replace multiple
181 \"blacklist_from\" entries with a single wildcard entry such as:
183 blacklist_from *@*amazingoffersdirect2u.com
185 In versions of SpamAssassin (2.50 and on) that support a Bayesian
186 classifier, \\[mh-junk-blacklist] uses the program \"sa-learn\"
187 to recategorize the message as spam. Neither MH-E, nor
188 SpamAssassin, rebuilds the database after adding words, so you
189 will need to run \"sa-learn --rebuild\" periodically. This can be
190 done by adding the following to your crontab:
192 0 * * * * sa-learn --rebuild > /dev/null 2>&1"
193 (unless mh-spamassassin-executable
194 (error "Unable to find the spamassassin executable"))
195 (let ((current-folder mh-current-folder)
196 (msg-file (mh-msg-filename msg mh-current-folder))
198 (message "Reporting message %d..." msg)
199 (mh-truncate-log-buffer)
200 ;; Put call-process output in log buffer if we are saving it
201 ;; (this happens if mh-junk-background is t).
202 (with-current-buffer mh-log-buffer
203 (call-process mh-spamassassin-executable msg-file mh-junk-background nil
204 ;;"--report" "--remove-from-whitelist"
205 "-r" "-R") ; spamassassin V2.20
206 (when mh-sa-learn-executable
207 (message "Recategorizing message %d as spam..." msg)
208 (mh-truncate-log-buffer)
209 (call-process mh-sa-learn-executable msg-file mh-junk-background nil
210 "--single" "--spam" "--local" "--no-rebuild")))
211 (message "Blacklisting sender of message %d..." msg)
212 (with-current-buffer (get-buffer-create mh-temp-buffer)
214 (call-process (expand-file-name mh-scan-prog mh-progs)
216 (format "%d" msg) current-folder
217 "-format" "%<(mymbox{from})%|%(addr{from})%>")
218 (goto-char (point-min))
219 (if (search-forward-regexp "^\\(.+\\)$" nil t)
221 (setq sender (match-string 0))
222 (mh-spamassassin-add-rule "blacklist_from" sender)
223 (message "Blacklisting sender of message %d...done" msg))
224 (message "Blacklisting sender of message %d...not done (from my address)" msg)))))
227 (defun mh-spamassassin-whitelist (msg)
228 "Whitelist MSG with SpamAssassin.
230 The \\[mh-junk-whitelist] command adds a \"whitelist_from\" rule to
231 the \"~/.spamassassin/user_prefs\" file. If the \"sa-learn\" command
232 is available, the message is also recategorized as ham.
234 See `mh-spamassassin-blacklist' for more information."
235 (unless mh-spamassassin-executable
236 (error "Unable to find the spamassassin executable"))
237 (let ((msg-file (mh-msg-filename msg mh-current-folder))
238 (show-buffer (get-buffer mh-show-buffer))
240 (with-current-buffer (get-buffer-create mh-temp-buffer)
242 (message "Removing spamassassin markup from message %d..." msg)
243 (call-process mh-spamassassin-executable msg-file t nil
245 "-d") ; spamassassin V2.20
247 (kill-buffer show-buffer))
248 (write-file msg-file)
249 (when mh-sa-learn-executable
250 (message "Recategorizing message %d as ham..." msg)
251 (mh-truncate-log-buffer)
252 ;; Put call-process output in log buffer if we are saving it
253 ;; (this happens if mh-junk-background is t).
254 (with-current-buffer mh-log-buffer
255 (call-process mh-sa-learn-executable msg-file mh-junk-background nil
256 "--single" "--ham" "--local" "--no-rebuild")))
257 (message "Whitelisting sender of message %d..." msg)
259 (car (mh-funcall-if-exists
260 ietf-drums-parse-address (mh-get-header-field "From:"))))
262 (unless (or (null from) (equal from ""))
263 (mh-spamassassin-add-rule "whitelist_from" from))
264 (message "Whitelisting sender of message %d...done" msg))))
266 (defun mh-spamassassin-add-rule (rule body)
267 "Add a new rule to \"~/.spamassassin/user_prefs\".
268 The name of the rule is RULE and its body is BODY."
269 (save-window-excursion
270 (let* ((line (format "%s\t%s\n" rule body))
272 (file (expand-file-name "~/.spamassassin/user_prefs"))
273 (buffer-exists (find-buffer-visiting file)))
275 (if (not (search-forward (format "\n%s" line) nil t))
277 (goto-char (point-max))
278 (insert (if (bolp) "" "\n") line)
280 (if (not buffer-exists)
281 (kill-buffer nil)))))
284 (defun mh-spamassassin-identify-spammers ()
285 "Identify spammers who are repeat offenders.
287 This function displays a frequency count of the hosts and domains
288 in the \"blacklist_from\" entries from the last blank line in
289 \"~/.spamassassin/user_prefs\" to the end of the file. This
290 information can be used so that you can replace multiple
291 \"blacklist_from\" entries with a single wildcard entry such as:
293 blacklist_from *@*amazingoffersdirect2u.com"
295 (let* ((file (expand-file-name "~/.spamassassin/user_prefs"))
296 (domains (make-hash-table :test 'equal)))
298 ;; Only consider entries between last blank line and end of file.
299 (goto-char (1- (point-max)))
300 (search-backward-regexp "^$")
301 ;; Perform frequency count.
303 (while (search-forward-regexp "^blacklist_from\\s-*\\(.*\\)@\\(.*\\)$"
305 (let ((host (match-string 2))
307 ;; Remove top-level-domain from hostname.
308 (setq host (cdr (reverse (split-string host "\\."))))
309 ;; Add counts for each host and domain part.
311 (setq value (gethash (car host) domains))
312 (setf (gethash (car host) domains) (1+ (if (not value) 0 value)))
313 (setq host (cdr host))))))
316 (delete-other-windows)
317 (pop-to-buffer (get-buffer-create "*MH-E Spammer Frequencies*"))
319 (maphash '(lambda (key value) ""
321 (insert (format "%s %s\n" key value))))
323 (sort-numeric-fields 2 (point-min) (point-max))
324 (reverse-region (point-min) (point-max))
325 (goto-char (point-min))))
329 ;; Bogofilter Interface
331 (defvar mh-bogofilter-executable (executable-find "bogofilter"))
334 (defun mh-bogofilter-blacklist (msg)
335 "Blacklist MSG with bogofilter.
337 Bogofilter is a Bayesian spam filtering program. Get it from your
338 local distribution or from http://bogofilter.sourceforge.net/.
340 Bogofilter is taught by running:
342 bogofilter -n < good-message
344 on every good message, and
346 bogofilter -s < spam-message
348 on every spam message. This is called a full training; three other
349 training methods are described in the FAQ that is distributed with
350 bogofilter. Note that most Bayesian filters need 1000 to 5000 of each
351 type of message to start doing a good job.
353 To use bogofilter, add the following recipes to \".procmailrc\":
355 MAILDIR=$HOME/`mhparam Path`
357 # Fight spam with bogofilter.
359 | bogofilter -3 -e -p
362 * ^X-Bogosity: Yes, tests=bogofilter
366 * ^X-Bogosity: Unsure, tests=bogofilter
369 If bogofilter classifies a message incorrectly, or is unsure, you can
370 use the MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist]
371 to update bogofilter's training.
373 The \"Bogofilter FAQ\" suggests that you run the following
374 occasionally to shrink the database:
376 bogoutil -d wordlist.db | bogoutil -l wordlist.db.new
377 mv wordlist.db wordlist.db.prv
378 mv wordlist.db.new wordlist.db
380 The \"Bogofilter tuning HOWTO\" describes how you can fine-tune Bogofilter."
381 (unless mh-bogofilter-executable
382 (error "Unable to find the bogofilter executable"))
383 (let ((msg-file (mh-msg-filename msg mh-current-folder)))
384 (mh-truncate-log-buffer)
385 ;; Put call-process output in log buffer if we are saving it
386 ;; (this happens if mh-junk-background is t).
387 (with-current-buffer mh-log-buffer
388 (call-process mh-bogofilter-executable msg-file mh-junk-background
392 (defun mh-bogofilter-whitelist (msg)
393 "Whitelist MSG with bogofilter.
395 See `mh-bogofilter-blacklist' for more information."
396 (unless mh-bogofilter-executable
397 (error "Unable to find the bogofilter executable"))
398 (let ((msg-file (mh-msg-filename msg mh-current-folder)))
399 (mh-truncate-log-buffer)
400 ;; Put call-process output in log buffer if we are saving it
401 ;; (this happens if mh-junk-background is t).
402 (with-current-buffer mh-log-buffer
403 (call-process mh-bogofilter-executable msg-file mh-junk-background
408 ;; Spamprobe Interface
410 (defvar mh-spamprobe-executable (executable-find "spamprobe"))
413 (defun mh-spamprobe-blacklist (msg)
414 "Blacklist MSG with SpamProbe.
416 SpamProbe is a Bayesian spam filtering program. Get it from your local
417 distribution or from http://spamprobe.sourceforge.net.
419 To use SpamProbe, add the following recipes to \".procmailrc\":
421 MAILDIR=$HOME/`mhparam Path`
423 # Fight spam with SpamProbe.
425 SCORE=| spamprobe receive
428 | formail -I \"X-SpamProbe: $SCORE\"
434 If SpamProbe classifies a message incorrectly, you can use the
435 MH-E commands \\[mh-junk-blacklist] and \\[mh-junk-whitelist] to
436 update SpamProbe's training."
437 (unless mh-spamprobe-executable
438 (error "Unable to find the spamprobe executable"))
439 (let ((msg-file (mh-msg-filename msg mh-current-folder)))
440 (mh-truncate-log-buffer)
441 ;; Put call-process output in log buffer if we are saving it
442 ;; (this happens if mh-junk-background is t).
443 (with-current-buffer mh-log-buffer
444 (call-process mh-spamprobe-executable msg-file mh-junk-background
448 (defun mh-spamprobe-whitelist (msg)
449 "Whitelist MSG with SpamProbe.
451 See `mh-spamprobe-blacklist' for more information."
452 (unless mh-spamprobe-executable
453 (error "Unable to find the spamprobe executable"))
454 (let ((msg-file (mh-msg-filename msg mh-current-folder)))
455 (mh-truncate-log-buffer)
456 ;; Put call-process output in log buffer if we are saving it
457 ;; (this happens if mh-junk-background is t).
458 (with-current-buffer mh-log-buffer
459 (call-process mh-spamprobe-executable msg-file mh-junk-background
465 ;; indent-tabs-mode: nil
466 ;; sentence-end-double-space: nil
469 ;; arch-tag: 603335f1-77ff-4306-8828-5d3dad51abe1
470 ;;; mh-junk.el ends here