lamson-1.0pre11/ 0000755 0000765 0000024 00000000000 11313464575 013055 5 ustar zedshaw staff lamson-1.0pre11/bin/ 0000755 0000765 0000024 00000000000 11313464573 013623 5 ustar zedshaw staff lamson-1.0pre11/bin/lamson 0000755 0000765 0000024 00000000222 11200524130 015013 0 ustar zedshaw staff #!/usr/bin/env python
from lamson import args, commands
import sys
args.parse_and_run_command(sys.argv[1:], commands, default_command="help")
lamson-1.0pre11/build.vel 0000644 0000765 0000024 00000005472 11313463463 014667 0 ustar zedshaw staff # Copyright (C) 2008 Zed A. Shaw. You're free to reuse this file
# in your build scripts in anyway and remove the copyright
# notice.
imports [
recipe(from 'scripts/testing' as 'testing')
recipe(from 'scripts/dist' as 'dist')
recipe(from 'scripts/sample' as 'sample')
]
options(
project "lamson"
default 'tests'
sudo 'sudo'
version '1.0pre11'
website './doc/lamsonproject.org/output/releases/'
version.file 'lamson/version.py'
setup(
name 'lamson'
version '1.0pre11'
author 'Zed A. Shaw'
description 'Lamson is a modern Pythonic mail server built like a web application server.'
url 'http://pypi.python.org/pypi/lamson'
download_url 'http://pypi.python.org/pypi/lamson'
author_email 'zedshaw@zedshaw.com'
package_data ( lamson [ 'data/prototype.zip' ])
packages ['lamson' 'lamson.handlers']
scripts ['bin/lamson']
install_requires ['chardet' 'jinja2' 'mock' 'nose' 'python-daemon']
)
)
depends(
build ['tests' 'prototype' 'examples' 'virus' 'version.gen' 'dist.install' 'dist.sdist']
commit ['dist.gen.setup' 'parser' 'dist.clean']
tests ['parser' 'testing.run']
release ['build' 'dist.release' 'book.release' ]
clean ['dist.clean']
pypi ['build']
)
targets(
commit [
$ bzr log --short > CHANGES
$ bzr commit
$ bzr push
]
virus [
$ clamscan -i -r
]
examples [
$ rm -rf run/queue examples/osb/run/* examples/librelist/run/*
$ rm -rf examples/librelist/app/data/archive/*
$ find . -name "main.db" -exec rm {} \;
$ find . -name "*.log" -exec rm {} \;
$ find . -name "*.err" -exec rm {} \;
$ find . -name "*.out" -exec rm {} \;
$ find . -name "*.pyc" -exec rm {} \;
$ find . -name "*.sw*" -exec rm {} \;
$ rm -f tests/test.db
$ rm -f tests/sbdb
$ rm -rf doc/lamsonproject.org/output/docs/api
]
prototype [
$ rm -f lamson/data/prototype.zip
$ cd lamson/data/prototype ; zip -r ../prototype.zip .
]
version.gen [
$ fossil info | grep checkout > /tmp/lamson.rev
py [
|rev = open("/tmp/lamson.rev").read().split()[1]
|ver = {"version": version, "rev": [rev[:10], rev]}
|open("%(version.file)s", 'w').write(
| "VERSION=" + repr(ver))
]
]
coverage [
$ nosetests --quiet --with-coverage --cover-package lamson
]
pypi [
$ python setup.py register sdist bdist_egg upload
]
)
lamson-1.0pre11/doc/ 0000755 0000765 0000024 00000000000 11313464573 013620 5 ustar zedshaw staff lamson-1.0pre11/doc/lamsonproject.org/ 0000755 0000765 0000024 00000000000 11313464573 017266 5 ustar zedshaw staff lamson-1.0pre11/doc/lamsonproject.org/build.vel 0000644 0000765 0000024 00000001312 11217111635 021061 0 ustar zedshaw staff imports [
module(from "vellum.commands")
]
options(
default 'sync'
sudo 'sudo'
)
depends(
sync ['gen']
)
targets(
commit
$ bzr commit
gen [
$ python webgen.py
$ cd ../.. && epydoc --graph classtree --url http://lamsonproject.org/ --name "Lamson Mail Server" --html --redundant-details lamson -o doc/lamsonproject.org/output/docs/api/
]
sync [
$ rsync -av input/* output/
$ rsync -avuz output/* zedshaw@www.zedshaw.com:/usr/local/nginx/html/lamsonproject/
$ rsync -av ../../dist/* zedshaw@www.zedshaw.com:/usr/local/nginx/html/lamsonproject/releases/
]
)
lamson-1.0pre11/doc/lamsonproject.org/ChangeLog 0000644 0000765 0000024 00000003747 11310773564 021053 0 ustar zedshaw staff === 2009-12-12 ===
20:00:38 [42768ade2b] *CURRENT* 1.0pre8 release announced and done. (user:
zedshaw tags: trunk, 1.0pre8)
19:43:53 [bb1b0a0dae] Updated MailBase.__nonzero__ to include parts as well.
(user: zedshaw tags: trunk)
19:40:51 [2916b83307] Created ChangeLog from fossil timeline of commits.
(user: zedshaw tags: trunk)
19:36:14 [41530d8a34] Version bump to 1.0pre8 for release. (user: zedshaw
tags: trunk)
19:31:21 [dfe158b282] Initial support of using Delivered-To or similar more
exact To header. (user: zedshaw tags: trunk)
19:14:13 [e9988679b7] Updated prototype.zip for upcoming release. (user:
zedshaw tags: trunk)
19:13:51 [98e94b6103] Potentially fixed the unicode/ascii issues with shelve
storage of confirmations and routing. (user: zedshaw tags: trunk)
18:56:35 [6224ae68e4] Updated docs about confirmation to fix a doc bug. (user:
zedshaw tags: trunk)
18:56:14 [103454c43c] Make things clean up better on INT or TERM (not HUP).
(user: zedshaw tags: trunk)
18:25:29 [009451cabc] Added meta tags to the site so it can be found easier. I
didn't know search engines still use that crap. (user: zedshaw tags:
trunk)
18:22:24 [8908d7852d] Applied starttls, username, password patch to server.
(user: zedshaw tags: trunk)
18:16:01 [9a9f9f37b6] Applied patch for handling odd email addresses and white-
listing what headers have email addresses. (user: zedshaw tags: trunk)
18:06:06 [7605ef0517] Patch for queue bugs applied [bug 9d330c8f80] (user:
zedshaw tags: trunk)
18:02:50 [ef9808c4f7] Point at the right archive browser. (user: zedshaw tags:
trunk)
18:01:39 [3b3dc9391a] Better logging message on failure to connect to relay.
(user: zedshaw tags: trunk)
=== 2009-12-05 ===
18:05:40 [ddc4369b1d] Initial commit to the new fossil based support site.
(user: zedshaw tags: trunk)
17:48:22 [8140fcad03] initial empty check-in (user: lamson tags: trunk)
lamson-1.0pre11/doc/lamsonproject.org/config.py 0000644 0000765 0000024 00000001227 11225636102 021076 0 ustar zedshaw staff import os
author = 'Zed A. Shaw' # Default author name. Overridden in individual document
THIS_DIR = os.path.abspath(os.path.dirname(__file__))
input_dir = os.path.join(THIS_DIR, 'input')
output_dir = os.path.join(THIS_DIR, 'output')
template_dir = THIS_DIR
template = os.path.join(template_dir, 'template.html')
### Optional parameters
options = { 'baseurl':"", # if not set, relative URLs will be generated
'sitename':"Lamson Project(TM)",
'slogan': "Lamson The Python SMTP Server and Framework",
'extensions':['.txt'],
'format': 'text/x-textile',
'siteurl': 'http://lamsonproject.org',
}
lamson-1.0pre11/doc/lamsonproject.org/input/ 0000755 0000765 0000024 00000000000 11313464573 020425 5 ustar zedshaw staff lamson-1.0pre11/doc/lamsonproject.org/input/about.txt 0000644 0000765 0000024 00000003537 11210043226 022270 0 ustar zedshaw staff From: Zed
Title: About Lamson
Lamson started more than a year ago as a fun side project of "mine":http://zedshaw.com/
after a few evil experiences with sendmail and various mailing list software. I
realized that e-mail systems didn't have anything like a modern "framework" for building
applications. Rather than replicate the mess of aliases, pipes, processes, and nasty m4
macros that currently existed, I decided to write something different.
Lamson was originally called "Son Of Sam", but that name proved too difficult to work
with as a project name. Apparently people don't like their software named after serial
killers (actually, that was his dog's name).
"Lamson Tubes" is a colloquial name for Pneumatic Tubes which were used last century to
deliver mail, packages, and hazardous material to the corporate world. They are still
in use today.
Now Lamson is a fully functioning SMTP server, relay, proxy, and e-mail application framework.
It supports most RDBMS, has templates, and is easy to manage and deploy. Lamson is smarter
than most any e-mail processing system out there thanks to Python.
However, as great as Lamson is for processing email intelligently, it isn't the best solution
for delivering mail. There is 30+ years of SMTP lore and myth stored in the code of mail
servers such as "Postfix":http://www.postfix.org/ and "Exim":http://www.exim.org/ that would
take years to replicate and make efficient. Being a practical project, Lamson defers to much
more capable SMTP servers for the grunt work of getting the mail to the final recipient.
I currently use Lamson in my own work, but I'm always looking for people doing
interesting things with e-mail. Even if you're a spammer, or someone trying to
destroy the spammers, I want to hear from you.
If you've got something interesting, feel free to "contact me":/contact.html and
talk about it.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/ 0000755 0000765 0000024 00000000000 11313464573 021350 5 ustar zedshaw staff lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-05-16.txt 0000644 0000765 0000024 00000001460 11203540277 023144 0 ustar zedshaw staff title: Lamson Project Site Launched
Today I launched the Lamson Project site at
"lamsonproject.org":http://lamsonproject.org/ and started filling in the
content. Lamson is really turning into a fun and useful project, and hopefully
the site will get other people interested in it and using it.
I took the design from one of the many free web design sites and reused the same Python blog script that I use on "my own site":http://zedshaw.com/ so getting this up and running was cake.
Subscribe to the "RSS feed":/feed.xml and I'll soon be doing a blog post announcing mailing lists and other email services that I'll host on lamsonproject.org as well as new documentation and code drops.
If you browse around you'll find some useful documentation, but nothing complete just yet. I'm still writing most of it.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-05-18.txt 0000644 0000765 0000024 00000010144 11213057130 023135 0 ustar zedshaw staff Title: Bug Fix 0.8.4, Mailing Lists, Spam Blocking
A few announcements from my work on Lamson the last few days. I managed to fix
a bug, put Lamson to work doing Lamson's mailing lists, and use Lamson to do
some spam blocking on my own email account.
Hopefully eating my own dogfood won't be too painful.
h2. Grab 0.8.4, Important Bug Fix
I wasn't using the SMTPServer class in Python correctly, and was stopping the
whole server when a single channel had an error. It's a one line fix and I've
been running it for a few days with no problems.
Please make sure that you upgrade if you installed Lamson:
sudo easy_install --upgrade lamson
Or you can "grab it from PyPI":http://pypi.python.org/pypi?name=lamson&version=0.8.4&:action=display
h2. Mailing Lists
Lamson is now running its very simple _examples/mailinglist_ sample on
lists.lamsonproject.org. You can send an email to:
lamson.users-subscribe@lists.lamsonproject.org
And try it out. The software running that is also what's available in the
_examples/mailinglist_ sample (minus a few tweaks) so if you want to help make it
a real functioning mailing list then feel free to contribute.
I'm subscribed there, and I'll be working on it over the next week to turn it
into a more complete and robust mailing list. An important thing I need to
implement is bounce and vacation detection. I'll probably also take my spam
filter hacks and work them in as well.
Also, the software at lists.lamsonproject.org is an open mailing list system.
Feel free to use it at your own risk, and if it becomes popular then I may turn
it into a permanent thing.
As an open mailing list system, what it does is creates any list that doesn't
exist when you subscribe. Think of it as lazy loading for mailing lists. No
need to fill out forms, beg for permission, alter aliases, edit text files, or
run a ton of shell commands.
Just subscribe and go.
Now, if that turns into a massive abuse vector then I'll turn it off, but I'd
like to make it work for people since it demonstrates something simple that
Lamson can do which other mailing list systems are bad at.
h2. Spam Filtering And Graylists In The Works
My personal email account (zedshaw@zedshaw.com) is now running
"Spambayes":http://spambayes.sourceforge.net/ inside of Lamson to filter spam.
Here's the code (stripped down for show):
class SpamHandler(server.MessageHandler):
"""Uses app.spam to mark all messages it receives as Spam."""
def process(self, session, message, args):
filter = spam.Filter()
filter.train_spam(message.msg)
filter.close()
class FilterHandler(server.MessageHandler):
"""
Uses the spam filter to either queue up a message
into the run/queue for later review, or forward
it on if it is not spam.
"""
def process(self, session, message, args):
classifier = spam.Filter( )
classifier.filter(message.msg)
if message.msg['X-Spambayes-Classification'].startswith('spam'):
q = queue.Queue('run/spam')
q.push(str(message))
else:
self.relay.deliver(message)
This shows Lamson's raw "handler style" of extension, which the Finite State Machine
stuff is built on. These are wired into the config/settings.py file routing
so that any mail that's sent to zedshaw@zedshaw.com goes through *FilterHandler*
and any mail that sent to bogus addresses @zedshaw.com goes through *SpamHandler*.
It turns out that a lot of spam is delivered to random addresses at zedshaw.com, so
this works as an autotraining mechanism. Well, at least until some asshat figures
out he can game my spam filter.
The FilterHandler then takes messages that SpamBayes knows is spam and puts it into
a queue for later review. I periodically check that folder and pull out any false
positives for retraining.
It's all very hacky right now, but seems to be working really well, so once I work out
the full workflow I'll formalize this code and add it to Lamson as extra handlers
to use. I'll also implement a simple graylisting system and include that too.
Well, enjoy the work so far and shoot me any comments you have.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-05-19.txt 0000644 0000765 0000024 00000002741 11204561532 023150 0 ustar zedshaw staff Title: New Site Look, Same Great Content
This is just a quick update to say thanks to "Ken Keiter":http://kenkeiter.com/
for creating a new "lamsonproject.org":http://lamsonproject.org/ site layout and
design. The new site should be easier to read, have more breathing room, and look
easier on the eyes. It's even got a logo:
!http://lamsonproject.org/images/lamson.png!
Which I like quite a lot.
Feel free to shoot your comments about the design to "me.":/contact.html
h2. Two Days Of Spam Filtering
So far the new Lamson based spam blocking and filtering is working pretty well.
The majority of spam is blocked, however the success rate of
"SpamBayes":http://spambayes.sourceforge.net/ needs to improve. Right now I've
trained it with a large set of ham, and with a reasonable number of spam
messages as I encounter them. I think it's received about a 20/1 (ham/spam)
ratio for training.
With that training it's sorted about 50 spam, no false positives, and missed
about 15 spam as "unsure", with about 4 as "ham". The 4 that were classified
as ham were just one liners with a link in them. The unsure spam probably
could have just been treated as spam, but I'm being conservative while I test
it out.
Hopefully the success rate improves over the next few days. SpamBayes is
great for being a Python library I can use easily, but if it's success rate
doesn't get much higher then I may look at other options. Of course, there
are more important things I could be doing right now too.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-05-20.txt 0000644 0000765 0000024 00000000404 11205152004 023121 0 ustar zedshaw staff Title: Lamson Project Ideas
I wrote a "blog post about project ideas for Lamson":http://zedshaw.com/blog/2009-05-20.html
on my personal blog. Head on over if you're looking for something to hack on, or just
want something to read that isn't about the web.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-05-24.txt 0000644 0000765 0000024 00000012322 11213057130 023132 0 ustar zedshaw staff Title: Features For The 0.9 Release (Soon)
Content-Type: text/html
I've been hard at work cooking up the very nice new routing system, and
I must say it is rather tasty. I've gone and created a whole new
routing and state management design that uses decorators right in your
handler modules to indicate how each state will expect mail addresses.
For the 0.9 release happening tomorrow I've got a new Router setup,
spam filtering baked in nice and clean, improved test coverage, and
indirect state storage for those who don't like SQLAlchemy.
New Routing Decorators
The new routing design is very nice if I do say so myself. It should
reduce the amount of Python regex wizardry you need to learn, help set
reasonable defaults, and put your handlers and routing in one spot so
they are easier to maintain. In addition the new design eliminated many
files and flaws from the previous design.
The formatting on the @route decorator will hopefully simplify the use
of regular expressions. Rather the above route for
"(list_name)-(action)@(host)" gets translated by Lamson into
'^(?P[a-zA-Z.0-9]+)-(?P[a-zA-Z0-9]+)@(?Ptest.com)$'
which is a hair pulling monstrosity. It also is using defaults so that you
only have to indicate basic formatting of the email and leave the regexes to
the main configuration file.
This new setup also has some interesting implications in how you can use it.
For example, you can register multiple address forms for each state (UNKNOWN, END, etc.)
so that it can handle similar activity.
Spam Filtering Per State
Now that the routing is nice and generic, I could implement a decorator for
using SpamBayes to filter spam to your state functions. If you don't
want a particular state to receive spam then you just use the @spam_filter
decorator:
from lamson.routing import route, route_like
from lamson.spam import spam_filter
ham_db = "tests/sddb"
@route("(anything)@(host)", anything=".+", host=".+")
@spam_filter(ham_db, "tests/.hammierc", "run/queue")
def START(message, **kw):
print "Ham message received. Going to END."
return END
@route_like(START)
def END(message, *kw):
print "Done."
This simply attaches the @spam_filter decorator to START and then when START
runs it transitions to END. It's still a little rough since you're having to
configure the @spam_filter right there, rather than in a config/settings.py file,
but it works. Later versions will be much cleaner.
Indirect State Storage
I've also abstracted away the state storage, which will open the door to
backends in any kind of storage you want, not just SQLAlchemy. The only thing you'll
need to implement to use a different storage is a simple get/set/clear set of functionality
and then attach it to the Router class to make it use the new storage.
Currently the code is using a simple MemoryStorage class to speed up the unit test runs,
and then I'll implement the SQLAlchemy store and probably a Tokyo Tyrant store too.
Higher Test Coverage
I also worked on getting the unit test coverage up as high as I could. Since Lamson is a
server with nasty OS level things like sockets and daemons, it is difficult to fully test
in just simple unit tests. It's at about 85% right now with only the daemon and server
commands not being tested.
Finally, this new design gets rid of the old class based handlers, the Conversation style
of Finite State Machine handlers, and quite a bit of code that was design cruft from
my previous iterations. It should be much cleaner and meaner, and will have a nicer
experience for developers.
Hold On Though
The code in the bzr branch is still in a state of flux, so feel free to grab it and look, but
you probably don't want to actually do anything with it. I should have all the samples
converted to the new style and then I'll do a 0.9 release. It should happen tomorrow (May 25th).
I'll have another blog post and some documentation for the new stuff tomorrow and all next week.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-05-28.txt 0000644 0000765 0000024 00000005643 11210151522 023143 0 ustar zedshaw staff Title: 0.9-pre2 Up For Testing, Docs Too
First off, my apologies to everyone if your RSS reader went crazy today.
I include documentation changes in the RSS feed so that people can
easily track updates to the Lamson docs. However, that means when I'm
writing a lot of documentation it hits the feed repeatedly.
The good news though is that you can now get at the 0.9-pre2 release of
Lamson from the "releases":/releases/ page and you can read the great
new documentation I've got.
Simply go to the "documentation":/docs/ list and go through. You'll
want to check out "how to install Lamson in a
virtualenv":/docs/lamson_virtual_env.html by Zachary Voase. You'll
also want to read through the current draft of the "getting
started":/docs/getting_started.html as it covers all the nitty gritty
things you need.
h2. Migrating To 0.9
I'm getting the 0.9 pre-releases out so that people can start
migrating and feeding me problems they encounter.
I'll have a full list of changes and some documentation on
migrating to 0.9. The main things that changed are covered
in the "getting started":/docs/getting_started.html documentation,
notably how Lamson is configured and how routing works. Routing
is the biggest change of all so read through it.
I'll have documentation on exactly how Routing works and
how to use it too coming soon.
If you started using Lamson at the 0.8.x level and need help
migrating "contact me":/contact.html and I'll help you out.
Migration should involve just tagging your state functions with
the right route decorators.
h2. Mailing List Is Dead, Long Live OSB
I really didn't like the way I did the mailinglist example, so
rather than keep it alive I just killed it and will rewrite it
after 0.9. In its place is the _examples/osb_ application which
is a One-Shot-Blog. You send it an email and it makes a blog
page and index for it. It's simple enough for everyone to
comprehend and I'll be able to cover it in documentation easily.
The OSB sample has all the right stuff for you to see how all of
Lamson's features work, so check it out and use it as the basis
for your own applications.
You can get it by "downloading the source":/releases/ and looking
in the examples/osb directory.
h2. 0.9 Probably This Weekend
I want to get all the documentation done for the 0.9 release
and finish the *very* last feature that it needs. I'm hoping
to have that in the next few days, at least by Monday.
The only feature remaining for 0.9 is a SQLAlchemy based state
storage, and maybe a shelf based one. Lamson now stores its
state in an abstract "StateStorage":/docs/api/lamson.routing.StateStorage-class.html class that lets you store state however you need.
Right now all that's implemented is a "MemoryStorage":/docs/api/lamson.routing.MemoryStorage-class.html which actually works well enough
for most applications that you just start off, and is great for
unit testing. But, there needs to be a permanent state store
support.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-05-31.txt 0000644 0000765 0000024 00000002037 11210533442 023134 0 ustar zedshaw staff Title: Lamson 0.9 Later Today
I have been working hard on the documentation and scrubbing the code for Lamson and the 0.9 release
coming out soon. The only things I feel I need to do before an official 0.9 release are:
# Clean up the one-shot-blog demo application a bit more.
# Write a small set of instructions on writing your own "StateStorage.":http://lamsonproject.org/docs/api/lamson.routing.StateStorage-class.html
# Do a last final code review to check for any obvious problems.
Once I do that I'll make 0.9 official. If you're using 0.8.x right now, or just playing with
Lamson, please go grab the "0.9 pre-release":/releases/ and make sure it works.
There is one bug in that 0.9-pre3 release where the router won't properly route of the form:
To: "First Last"
This is fairly rare, since someone would have to craft an email like this, and it would just
be ignored anyway. *That bug is fixed in the current 0.9 source.*
Please shoot me any bugs you find and I'll work them into the 0.9 release.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-06-01.txt 0000644 0000765 0000024 00000007375 11210662772 023155 0 ustar zedshaw staff Title: Lamson 0.9 Is Out, Find My Bugs!
I just pushed Lamson 0.9 up to PyPI for everyone to grab and break. This release
features a complete redesign of the routing, state handling, templating, and a full
set of very complete documentation. Everyone who was using 0.8.x series should be able
to migrate to this version with some work, but it won't be terribly painful (assuming
you have unit tests).
h2. Getting 0.9
Easiest way to install is with "easy_install":http://peak.telecommunity.com/DevCenter/EasyInstall
straight from PyPI:
sudo easy_install lamson
There's also instructions on "installing to a virtualenv":/docs/lamson_virtual_env.html
for those who like that.
Best thing to do is read the "Getting Started":/docs/getting_started.html instructions
and when you're done with at least the 30 second introduction you'll have your Lamson.
Try to go through the whole document if you can and send me feedback.
h2. Big Changes
The biggest change is that the routing mechanism and internals are totally different.
Now instead of putting all your routes into the config/settings.py as a big array,
you just put them on each state function as a decorator. To help you debug these
fancy routes there's also a new @lamson routes@ command that will dump them in
their final state.
The biggest impact of the new design is the old school "BlahHandler" classes are
gone. Later versions of Lamson might bring back the ability to use a class as a
handler.
After that comes the removal of depending on SQLAlchemy for state storage. You
can now "write your own state storage":/docs/writing_a_state_storage.html and
store things however you need to whatever you need. There is currently a simple
storage for storing in memory and for using a Python shelve dict.
Next there was a change to how Lamson boots up and configures itself giving you
more control over what it uses, and giving your handlers better access to your
gear. This change also let's you use better unit tests that don't require a
running lamson server, as well as giving you the old "integration test" style
tests that hit your real server.
There's also tentative support for PyEnchant spell checking of your templates,
and a new @lamson spell@ command to help you use it. See the "getting started":/docs/getting_started.html
document for how to configure that, but be warned that PyEnchant is kind of a
mean install.
Finally, I switched to Jinja2 templates by default, but you can use Mako fairly easily
by altering a configuration variable in the boot.py.
h2. Look At All The Shiny Docs
In addition to redesigning the guts of Lamson so that it's easier to work on and use,
I also spent a lot of time cranking out good documentation so people can get started
using it. Take a look in the "documentation":/docs section and have a gander
at the "API documentation":/docs/api/ that is generated from the source documentation.
h2. New Example: One-Shot-Blog
In the source distribution "you can grab here":/releases/ there's an example
of doing a simple blog that uses just email to manage the blogger's posts.
The example replaces the mailing list example (for now) as it is simpler and
doesn't use a database.
h2. Please Break It
The 0.9 release is the result of a lot of work, but I want people to break it and
criticize it to death. The goal is to have people beat up this basic functionality,
producing a series of 0.9.x releases, and then when everything is tight doing a 1.0
release.
h2. Thanks Everyone
I'd like to thank everyone on Twitter, Hacker News, and yes, even Reddit for
their comments both good and bad. It helped me make good decisions knowing
what people wanted, so keep the feedback coming.
Thanks also to those who donated documentation, praise, or other help directly
to the project.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-06-03-2.txt 0000644 0000765 0000024 00000005044 11211630350 023271 0 ustar zedshaw staff Title: Lamson 0.9.2, Test Coverage 97%
The 0.9.2 release is out and ready for everyone to easy_install. I spent the
day getting rid of my tech debt by boosting the Lamson test coverage to a
whopping 97%.
I also wrote new documentation on "Running A Queue Receiver":http://lamsonproject.org/docs/deferred_processing_to_queues.html in a separate process using Lamson.
h2. Test Coverage 97%
I used the very awesome "mock":http://www.voidspace.org.uk/python/mock/ library
which let me mock out a bunch of the system calls that the "lamson.commands":http://lamsonproject.org/docs/api/lamson.commands-module.html
module accesses to do its work.
By mocking out things like @sys.exit@ and sockets I'm able to run the code and
make sure those methods were called correctly. I could also throw exceptions
to make sure that exception handlers ran correctly.
h2. Two Bugs Fixed
Interestingly enough, increasing the test coverage to 97% didn't find any more
bugs. I did find one small bug and one obvious bug in the 0.9.1 release.
# If you gave a trailing option that wasn't a string it would throw an exception.
# If you tried to start with a different boot script then it wouldn't do it.
These are now fixed in the 0.9.2 release, so make sure you "grab it":/download.html
with the easiest method being:
$ sudo easy_install lamson
h2. IRC Channel on irc.freenode.org
I'm now hanging out in the #lamson channel on irc.freenode.org. Anyone who wants to
come and chat about lamson can stop by. I should be there all the time, but I
may not respond immediately.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-06-03.txt 0000644 0000765 0000024 00000003623 11211415751 023141 0 ustar zedshaw staff Title: Lamson 0.9.1 Out, New Docs
I released Lamson 0.9.1 today so please "grab it and test it":/download.html and shoot me feedback.
The little changes are easier handling of attachments in MailRequest, a
small tweak to spam_filter, and then a bunch of test enhancements and extra
gear. The big change is a redesign to lamson.args so that it infers the
command line arguments from the lamson.commands function keyword arguments.
It's mostly an internal change since others can't write their own commands
(yet).
h2. New Documentation
I ran into a bunch of folks who didn't know that Lamson had built-in spam
filtering, so I wrote a document on "Filtering Spam With
Lamson":http://lamsonproject.org/docs/filtering_spam.html that tells you how
to use it.
Lamson's spam filtering support is fairly simplistic, but still useful for
filtering crap out of your handlers. It's biggest problem is the current
implementation requires you to use the (barely documented) "SpamBayes"
command line tools for things like training.
I've got some instructions to get you started with the most important
SpamBayes command line tool you'll need for training.
h2. ChangeLog
Here's the log of what I did:
* Big code review, cleanup, and test boosting.
* Modified the MailRequest.body so that it works with attachments.
* Updated the OSB in preparation for a redesign to make it more interesting.
* Didn't like the name 'attachments' for getting what's really all the parts of a message.
* Implemented a method on MailRequest to return all of the attachments in a message reliably.
* Setup the muttrc to use env to find the lamson command.
* Updated the commands and utils so that it works with the new lamson.args setup.
* Refactored lamson.args to use inspect to figure out the required defaults of a command function, thus removing the need for a...
* Contributed change to muttrc so it uses lamson wherever it is installed, even virtualenv.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-06-04.txt 0000644 0000765 0000024 00000004626 11211760244 023146 0 ustar zedshaw staff Title: OneShotBlog Sample (Hack) Running
I *finally* got off my ass and put the "OneShotBlog":http://oneshotblog.com/ sample up.
This is the code (plus a few little tweaks) from the sample that is in the "Lamson source":/releases/
running on another server. It's using all the features of Lamson, include the new "Queue Receiver":/docs/deferred_processing_to_queues.html
functionality. So far it's working great considering I've just been hacking on it on the side to try
out the usability of the Lamson "APIs":/docs/api/ and not really taken it seriously.
The concept of the oneshotblog is a little weird:
# You send an email to SOMETAG@oneshotblog.com, it can have textile markup.
## SOMETAG can be anything that's a tag (chars, ints, periods, no leading periods), like: i.want.my.blog@oneshotblog.com
# That is posted for you at http://oneshotblog.com/posts/YOURADDRESS/SOMETAG.html
## I *do* block attempts to use a bad path by whitelisting. If you send something messed up for the tag it's ignored.
# Then your post is summarized and tossed on the "front page":http://oneshotblog.com/ with the most recent 50 shown.
## This is done by a deferred queue receiver that indexes messages as it receives them.
# If you send to that same tag, it replaces it with the new contents (although the index will show the old stuff for a while).
There's some dumb bugs in it currently, like delete doesn't remove your message from the index and other
fun things, but so far it's running as expected given the work I put into it.
Anyway, feel free to try it out and shoot me feedback. Try sending really broken email too so that
I can see what Lamson does with it.
h2. Future Features Of OSB
Just for fun I'm gonna clean up OSB and make it official. I'll add features like commenting on a post via email, each
post is a little mailing list, and handling image and code attachments. For the code attachments I'll colorize them
with Pygments and put them up with the post.
h2. Mailing Lists and Failmail
I'll be bringing back the mailing list example soon, and I'll also make an address failmail@lamsonproject.org.
More on that later.
h2. Python 2.6 Support
If you're running Python 2.6 then you'll want to wait for the 0.9.3 release later today. The deployed OSB sample
at "oneshotblog.com":http://oneshotblog.com/ is running on Python 2.6, so I had to fix about 4 failing tests
regarding unicode issues and weirdness with python-daemonize.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-06-06.txt 0000644 0000765 0000024 00000020043 11213057130 023132 0 ustar zedshaw staff Title: Lamson 0.9.3 Is Out And Sexy As Hell
This release is the result of me working on my little
"oneshotblog.com":http://oneshotblog.com/ project while tweaking and refining
Lamson as I go. The end result is 0.9.3 didn't have a lot of big code changes,
but all the tiny little changes add up to a very nice release. The highlights
of this release are more secure server runs, better character encoding handling
for headers, various cleanups in how mail is queued, and fixes for Python 2.6
support.
My plan is to keep using this release, improving what is there currently and
only adding minimal features, and just making what is there tighter. The goal
is to *have a 1.0 release out* once I've used it on
"oneshotblog.com":http://oneshotblog.com/ and the new "lamsonproject.org
mailing lists":/lists/ application.
h2. Getting It (As Usual)
All you need to do to get lamson is read the "download instruction":/download.html
and then go through the "getting started":/docs/getting_started.html to try it out.
The *docs may be out of date with this release*, but I'm going through docs tomorrow.
h2. NYLUG Python Workshop Tuesday June 09 6:00PM-8:00PM
I'll be presenting Lamson at the "NYLUG Python Workshop Tuesday June 09 6:00PM-8:00PM":http://nylug.org/pipermail/nylug-announce/2009-June/000795.html
and helping people get started with it. If you wanted to try it but didn't want
to waste the time then come down and I'll hook you up.
h2. OneShotBlog Sample Is Live
Lamson includes the source to the "oneshotblog.com":http://oneshotblog.com/ project
in the @examples/osb@ directory. You can "grab the source tar.gz":/releases/
and look in there fore the source. You should also try out the sample site to help
test out Lamson, especially if you use a weird character set (since that's the biggest
weakness currently).
This release of OSB features a bunch of bug fixes, tons of tests that improve the code
coverage, the ability to comment on a post by sending an email, decent spam filtering
of the comments, ability to flag a comment as spam, and lots of little tweaks. It
uses *all* of the features Lamson has right now.
It still has some work to do, and I want to add the ability to send pictures and source
code (with syntax highlight).
h2. The ChangeLog
The change log for this release is (with the big ones bold):
* Cleaned up some stray printing and logging.
* Now prints a nicer message if args get a bad argument.
* *Managed to get a file object out of the maildir which removes the need to reconvert from a message.*
* A few minor test suite tweaks to get coverage on the most recent features.
* *Implemented a reorg of how the servers are started so that they can bind port 25 and then drop priv to a safer user.*
* *Initial support for an undeliverable queue and a forwarding handler that you can run in a queue receiver to deliver unhandled mail to the relay. Needs tests.*
* Using the new Router.UNDELIVERABLE_QUEUE feature to have bad mail go to a queue.
* Fixed the queue_command so that it prints what queue it is working with.
* Added a setting for the Router to tell it to dump undeliverable mail to a queue.
* *First cut at semi-automated header encoding conversion.*
* Initial support for sort-of-automated header decoding/encoding on MailRequest.
* Found a bug in clear_queue which made it not able to clear alternate queues.
* Small changes to fix a non-idiomatic has_key usage.
* Tweaked testing so that it resets the Router state when you do a client.begin() in unit tests.
* Tweaked the boot vs. test logging setup to make it nicer for both situations.
* *Since the Router is the primary code loading mechanism, it now uses its own separate logger that you can redirect to stderr to see loading errors during dev.*
* *Created a disconnected set of config testing gear so that it can test without relying on any examples. Also configures the logger using a config file and cleans up test output.*
* *Added a feature to the routes command to print out how each matching regex will match a given test address.*
* Added an exception handler for loading and reloading handlers so that you can see which ones are broke during development.
* Added a -test feature to lamson routes that let's you test an address to see what routes it matches.
* *Updated for Python 2.6 so the tests will run, not sure why they pass in Python 2.5 actually.*
* Modification to the mail parsing so that it forces 'ascii' encoding to keep email.message_from happy in Python 2.6
h2. Using The Drop Privilege
Lamson can now drop privilege after binding port 25. Previously it was too difficult
to make the python-daemonize library do it correctly, but after I reorganized some code
I was able to do the privilege drop manually to implement it. What this means is you can
run your email server and *stop being root* right after you bind port 25.
To do this, you'll just run your server with:
You technically don't have to fix the permissions, but if you use "queue receivers":/docs/deferred_processing_to_queues.html
then you'll need to change the permissions back. Lamson still creates a few files as root before
dropping to your given UID and GID, but hopefully it will stop doing this soon.
Obviously, you might want to know what your UID and GID is, but it's important that you know
what *Python* thinks it is:
Python 2.5.1 (r251:54863, Jan 13 2009, 10:26:13)
[GCC 4.0.1 (Apple Inc. build 5465)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import os
>>> os.getgid()
500
>>> os.getuid()
500
>>>
There's other ways to find it out, but that's the Python way.
h2. Using lamson routes
The @lamson routes@ command has improved a bit so that you can inspect your routing and
test out an email address to see how it would be sorted. Here's a sample run:
osb $ lamson routes -test test.blog@oneshotblog.com
2009-06-07 02:33:31,678 - root - INFO - Database configured to use sqlite:///app/data/main.db URL.
Routing ORDER: [... lots of regex here ...]
Routing TABLE:
---
... each regex and what state functions it maps ..
---
'^post-confirm-(?P[a-z0-9]+)@(?Poneshotblog\\.com)$': app.handlers.post.CONFIRMING
---
TEST address 'test.blog@oneshotblog.com' matches:
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.index.POSTING
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.post.START
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.post.POSTING
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
osb $
If you're working with Lamson this is incredibly helpful, because it tells you what
routes you have, what functions they call, and then it'll take an email address and
tell you all the routes that match it.
The output needs to be cleaned up, but for debugging this is nice.
h2. The lamson web Command
While working on "oneshotblog.com":http://oneshotblog.com/ I kept having to
open up the site that it generates, so I just threw a dirt simple HTTP server
into Lamson. Here's a sample of running it:
~ $ cd projects/lamson/examples/osb/
osb $ lamson web -basedir app/data/
Starting server on 127.0.0.1:8888 out of directory 'app/data/'
localhost - - [06/Jun/2009 12:31:11] "GET / HTTP/1.1" 200 -
localhost - - [06/Jun/2009 12:31:11] "GET /styles/reset.css HTTP/1.1" 200 -
localhost - - [06/Jun/2009 12:31:11] "GET /styles/main.css HTTP/1.1" 200 -
localhost - - [06/Jun/2009 12:31:18] code 404, message File not found
It just stays there running and printing to the console, so I just use
"GNU screen":http://www.gnu.org/software/screen/screen.html and leave it
running in one of the windows.
h2. As Usual Test And Report
Test this release out, and shoot me feedback. I'm hanging on irc.freenode.org
again in the #lamson room, so just stop in if you need help.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-06-08.txt 0000644 0000765 0000024 00000003355 11213316150 023143 0 ustar zedshaw staff Title: A Screencast And Docs On Deploying Lamson And OneShotBlog
I just finished writing some new documentation on "Deploying Lamson and OneShotBlog":/docs/deploying_lamson.html that
shows you how to install Lamson and all required software into a completely clean Python 2.6 and virtualenv configuration.
The instructions take you from nothing to a running "oneshotblog.com":http://oneshotblog.com/ installation that you can
play with and hack on. The process of doing Lamson deployments is still a little too rough for me, but these instructions
should get people started.
I also created the first screencast "teaching you how to do the same deployment":/videos/ but stopping at installing
"oneshotblog.com":http://oneshotblog.com to keep things short. It is basically me talking and typing into a terminal
window to 12 minutes showing you how to do it. You can also "download the video":http://www.archive.org/details/Lamsonproject.orgDeployLamsonInAVirtualenv
from "archive.org":http://archive.org/ to get a better viewing experience.
Being my first screencast I can say that it is very tedious. I'll be working on various strategies to make the
process more streamlined and easier since having to take a whole day for each one will drive me insane.
If you have suggestions on making the video better, "let me know":/contact.html and also if you know of tools to
help clean up video. One thing I know right away needs to improve is the sound quality. An apartment in NYC is just
too damn noisy for this kind of thing.
Enjoy, and don't forget that I'll be "presenting Lamson at the NYLUG Python workshop tomorrow June 9th at 6:00pm":http://nylug.org/pipermail/nylug-announce/2009-June/000795.html
so come on down if you want to get your Lamson working.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-06-09.txt 0000644 0000765 0000024 00000001020 11213542615 023136 0 ustar zedshaw staff Title: Lamson At NYLUG Python Workshop Today @ 6:00PM
Just a quick reminder that I'll be presenting "Lamson":http://lamsonproject.org/ to the
NYLUG Python Workshop today at 6:00PM. The event is at
the NY Public Library Hudson Park Branch, 66 Leroy St., NY NY 10014
in NYC and you can "find out more here.":http://nylug.org/pipermail/nylug-announce/2009-June/000795.html
I'll be showing people how to set up Lamson, get it installed,
do cool stuff with it, and then just answering questions and helping
people work with it.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-06-14.txt 0000644 0000765 0000024 00000021570 11215276254 023153 0 ustar zedshaw staff Title: The Mailocalypse Is Upon Us!
I'm currently polishing off the two final features before I start going for
the Lamson 1.0 release. I've been using Lamson to make a few little cute applications
and create one thing for a potential client, and so far I haven't had to change much
since 0.9.3. It's great so far and I hope that Lamson 1.0 will be a fun release.
The *big* thing that must improve though is handling character encodings in emails.
After spending a week or more trying to come up with an automated conversion scheme that
would honor all the encodings on the planet, I had a realization.
h2. Why Isn't All Mail UTF-8?
Remembering the purpose of Lamson as a *modern* mail server and framework, it finally
struck me as dumb that I'm trying to keep around encodings that were invented in the
pre-Unicode days. Every modern system understands and displays UTF-8, and apart from
some pissy attitudes from the Japanese about the Han unification, there's really no reason
that every email Lamson handles can't be "upgraded" to UTF-8.
As I thought about this more I started to like the idea. Take all mail Lamson handles, and
upgrade it to utf-8. Then inside Lamson you know that it's future proofed and ready to
be used by Python no matter what. Finally when the mail goes out, the conversion for sending
is simply to encode the email as proper UTF-8 encoding consistently.
However, I'm not the type of person to think any random idea I have is instantly valid because
I thought it up. Before I make a grand announcement that Lamson only deals with UTF-8 I better
know that it's going to have minimal impact on people using it. I made a similar decision with
Mongrel, where I chose *not* to accept clients that didn't follow the standard, and it turned
out to block a ton of security attacks for free.
What if Lamson's required conversion to UTF-8 could not only simplify Lamson mail processing,
but also prevent spam and security flaws? I decided to do some basic analysis and find out.
h2. This Is The Mailocalypse
I've decided that Lamson will *NOT* try to maintain the encodings it is given by a client.
It will consider all encodings suspect, and attempt to convert them to UTF-8 as a means
of cleaning and simplifying the mail it receives before it talks to other systems. Every
other system you'd most likely talk to is either UTF-8 capable or even requires it.
I'm calling this "The Mailocalypse" because it's funny, but it will also separate Lamson from
other systems, in potentially awesome or horrible ways.
To validate this premise, I created a small Python script that takes a Maildir or mbox and tries to
convert every email into a UTF-8 encoding. It's not trying to write the emails, just
load them and convert every single piece reliably to UTF-8. It then reports any emails
that have a problem with the conversion. After that I ran it on a bunch of mail I had
and got other people to run it and report their findings.
bq. "You can grab the mailocalypse.py script here.":/mailocalypse.py Improvements welcome.
The data I got from a small sample of the mail (randomly selected) was the following:
bq. all is the total number of messages in the mailbox (including bad). bad is the number that were bad.
spam is whether that box was a spam box or not (meaning all messages were classified as spam).
I also made sure that the characters showed up in web browsers and in various UTF-8
mail clients and terminal windows. Most of the conversion was reliable and very
fast for a quick script.
With the above data, there's two things you can notice right away, and which are confirmed with a quick
R session:
# Spam fails to convert more frequently than ham messages. It looks like failure to convert is a 0.03 significant indicator that the message is spam. I'd need more data to confirm that.
# Legit mail (ham) only has a 1.2% chance of failing to convert, and a quick eye-ball sample says those failures mail are mostly spam that wasn't classified right.
h2. Potential Problems
This analysis shows initially that converting mail to UTF-8 could work, but there are two problems
I see so far:
# I may not be doing the best conversion in that script.
# Encryption and signatures will screw this up.
For the first problem I'm posting the script and asking people to check it out and
let me know if there's problems. Run it on your stuff and shoot me the ALL/BAD numbers
when it's done. Check the code and see if you can improve it. Whatever you can to show
that converting to UTF-8 is a good *OR* bad idea.
A key design decision though will be that you never lose the original email, only that it
is converted for Lamson so you can work with it in your code without it blowing up. If the
mail can't convert, then it's bad email and should be rejected anyway. No point trying to
process it. For encrypted email or ones with signatures you would just run all the validation
on the original, or forward the original on, depending on your application.
For example, a mailing list would still want to convert all mail to UTF-8 for
processing, posting to web sites, spam filtering, and rejecting potentially bad
email. Signed and encrypted email would then just be forwarded on in original form,
or decoded and validated depending on the how the mailing list works.
h2. Advice And Criticism Welcome
Take the mailocalypse script (presented below for review) and try it out. Confirm that
it works on most of your mail, and send me the results. Simplest way is to send me
a message on twitter like:
bq. @zedshaw ALL 2340 BAD 17 spam yes #mailocalypse
And I'll tally the results and do a better analysis.
You are also encouraged to fill me in on what I'm missing in the idea. Realize that *part*
of the idea is to eliminate uncommon corner cases and to prove that a situation is common
using evidence. Try to follow the same model by running the script on your mail to prove
that it won't convert reliably or devise your own script to demonstrate your thoughts.
h2. The Codes
"You can grab the mailocalypse.py script here.":/mailocalypse.py Improvements welcome.
import email
from email.header import make_header, decode_header
from string import capwords
import sys
import mailbox
ALL_MAIL = 0
BAD_MAIL = 0
def all_parts(msg):
parts = [m for m in msg.walk() if m != msg]
if not parts:
parts = [msg]
return parts
def collapse_header(header):
if header.strip().startswith("=?"):
decoded = decode_header(header)
converted = (unicode(
x[0], encoding=x[1] or 'ascii', errors='replace')
for x in decoded)
value = u"".join(converted)
else:
value = unicode(header, errors='replace')
return value.encode("utf-8")
def convert_header_insanity(header):
if header is None:
return header
elif type(header) == list:
return [collapse_header(h) for h in header]
else:
return collapse_header(header)
def encode_header(name, val, charset='utf-8'):
msg[name] = make_header([(val, charset)]).encode()
def bless_headers(msg):
# go through every header and convert it to utf-8
headers = {}
for h in msg.keys():
headers[capwords(h, '-')] = convert_header_insanity(msg[h])
return headers
def dump_headers(headers):
for h in headers:
print h, headers[h]
def mail_load_cleanse(msg_file):
global ALL_MAIL
global BAD_MAIL
msg = email.message_from_file(msg_file)
headers = bless_headers(msg)
# go through every body and convert it to utf-8
parts = all_parts(msg)
bodies = []
for part in parts:
guts = part.get_payload(decode=True)
if part.get_content_maintype() == "text":
charset = part.get_charsets()[0]
try:
if charset:
uguts = unicode(guts, part.get_charsets()[0])
guts = uguts.encode("utf-8")
else:
guts = guts.encode("utf-8")
except UnicodeDecodeError, exc:
print >> sys.stderr, "CONFLICTED CHARSET:", exc, part.get_charsets()
BAD_MAIL += 1
except LookupError, exc:
print >> sys.stderr, "UNKNOWN CHARSET:", exc, part.get_charsets()
BAD_MAIL += 1
except Exception, exc:
print >> sys.stderr, "WEIRDO ERROR", exc, part.get_charsets()
BAD_MAIL += 1
ALL_MAIL += 1
mb = None
try:
mb = mailbox.Maildir(sys.argv[1])
len(mb) # need this to make the maildir try to read the directory and fail
except OSError:
print "NOT A MAILDIR, TRYING MBOX"
mb = mailbox.mbox(sys.argv[1])
if not mb:
print "NOT A MAILDIR OR MBOX, SORRY"
for key in mb.keys():
mail_load_cleanse(mb.get_file(key))
print >> sys.stderr, "ALL", ALL_MAIL
print >> sys.stderr, "BAD", BAD_MAIL
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-06-20.txt 0000644 0000765 0000024 00000010450 11217126716 023142 0 ustar zedshaw staff Title: Lamson 0.9.4 With Unicode Super Powers
Lamson 0.9.4 is out and it's sporting a completely rewritten and meticulously
crafted encoding system. With the new "lamson.encoding":http://lamsonproject.org/docs/api/lamson.encoding-module.html code Lamson can now decode nearly any nasty horrible encoded spam
or mail you hand it, turn it into pristine nice Python unicode strings, and
then output sweet clean ascii or utf-8 in a consistent way.
The purpose of this new encoding system is to make sure that Lamson
is giving your handlers the best input it can, based on the assumption
that the world is evil and Lamson will be handed utter garbage.
In order to pull this off, Lamson actually attempts to decode everything
it gets, and uses "chardet":http://chardet.feedparser.org/ to guess whenever
it can't. It goes far enough to make sure that 99.9% of legit mail gets through,
and the rest is usually spam. However, Lamson is so good at this conversion
now that it even does it properly on about 99% of spam, even more.
It doesn't matter what character encoding is used, if there's a mix of encodings
or even if something lies about the encoding or doesn't give one. Lamson can
figure this out and convert it anyway, and the rest is junk.
You can look at this "image of a spam inbox":/lamson_vs_spam.png and
"this image of a spam inbox":/lamson_vs_spam_2.png where you can see a
screen session showing this off. This is a split screen with a Mutt on top
showing badly formatted spam, and then a Mutt on the bottom with the
results of Lamson's cleansing. This is just running in iTerm, but looks
the same in most unicode enabled terminals and clients.
The interesting side effect of Lamson's decoding is that it undoes almost all
of the spam obfuscation techniques quickly and consistently.
h2. There Will Be Bugs
I've thrashed the hell out of this code and made sure that I really took
the time to get it right. I've ran it on thousands and thousands of real and
spam mail and tested the crap out of it with unit tests and fuzzing.
Still, this is new code handling email so there will be bugs in it. If
you run into an email that you think should parse correctly, send it to
me and I'll look at it.
h2. How It Works
All Lamson does is completely parse every header and figure out how to decode it,
but it assumes that the client is a liar. If the string decodes without errors
then Lamson is happy, but if it blows up then Lamson uses chardet to figure out
what the contents really are encoded as. If chardet can't figure it out, or if
it's still busted, then Lamson rejects that mail (which happens less than a fraction
of a percent of the time to real email).
Once this decoding process is finished Lamson has converted the email completely
into Python unicode only. You can work with it in a modern way without worrying
about the original codecs, and you can set your headers and attachments without
worrying about setting the codecs because Lamson will use the same tech to get
the encoding right.
When Lamson writes an email, it assumes that your text can be encoded as either
plain ASCII (as most headers are), or UTF-8. If it can't, then your text
probably can't be processed by Lamson anyway. Lamson will favor ASCII first since
it's easier, and then use UTF-8 for anything that can't be ASCII.
bq. During this conversion process anything that's *not* text is treated as raw binary
and just decoded as-is.
By doing this Lamson produces nice clean email that can be easily processed and
passed around, and which you can review and debug easier. It also turns Lamson
into a "modernizing" agent since it is producing the email it wants to see.
h2. Getting This Release
I had some issues pushing this release to PyPI, so let me know if it barfs on you.
Otherwise, you can hit the "download":/download.html section and grab the
gear.
h2. The New "Cleanse" Command
In order to make it easier for people to try this new cleansing system on real
email I've written a quick @lamson cleanse@ command. What this command will do is
take a mbox or Maildir, and run it through the washing machine, writing the results
to a different maildir. When it's done you can go look at the reasons why some
mail failed, how many failures, and you can actually open that maildir mail and
look at it with your client.
Try it out and report any errors you find.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-06-22.txt 0000644 0000765 0000024 00000007765 11220041004 023137 0 ustar zedshaw staff Title: 0.9.5 Almost There, But Stumped On Templates
Since the 0.9.4 release I've rewritten the main part of the decoding parser
so that it's much cleaner and handles more edge conditions. If there's one
word that defines what makes MIME horrible it would be "edge".
It's amazing the kind of stupid crap clients are allowed to send out in headers.
I've actually got test cases from Mutt where it breaks headers up into completely
different encodings across multiple lines for no damn reason at all. Here's a
test I had to write just to cover the one worst case I found:
def test_dumb_shit():
# this is a sample of possibly the worst case Mutt can produce
idiot = '=?iso-8859-1?B?SOlhdnkgTel05WwgVW7uY/hk?=\n\t=?iso-8859-1?Q?=E9?='
should_be = u'H\xe9avy M\xe9t\xe5l Un\xeec\xf8d\xe9'
assert_equal(encoding.header_from_mime_encoding(idiot), should_be)
If you can decode what that actually is and why it's retarded then you'll win a
prize. I'll give you a hint: @B != Q@.
The example of the above working is "on this page at oneshotblog.com":http://oneshotblog.com/posts/zedshaw@zedshaw.com/test.html so that's solved (for now).
h2. Now About Those Templates
I originally wanted to make writing templates as easy as writing an email or a web page,
so I let you do things like this:
To: {{ message['from'] }}
From: {{ confirm_address }}@{{host}}
Subject: Please confirm your new blog {{ post_name }}
Reply-To: {{ confirm_address }}@{{host}}
Hi there, you requested that I create a blog with a post named:
{{ post_name }} : {{ message['subject'] }}
...
This is a "Jinja2":http://jinja.pocoo.org/2/ that "oneshotblog.com":oneshotblog.com sends
out when it makes you confirm creating a blog.
This seems all normal, since it's the headers you want and the body,
like an email, but it is *a total fantasy*. The reason is you can't actually
decode the above template if any part of it has non-ascii chars. I'll explain, and
then hopefully I can get some suggestions on what people would like.
First, Lamson does the right thing and in your templates you are getting perfectly
usable and future proof @unicode@ objects to work with in your handlers. Also,
Jinja2 has no problem with taking this unicode and processing the above template.
It works fine, and Jinja2 or Mako aren't to blame for the problem with
templates now.
The problem is actually in how email has to be parsed, since the email library
expects you to hand it a @ascii@ encoded string, no unicode allowed. Nothing else.
Lamson's new code can handle the unicode fine, the problem is actually in
this one function:
The above function has to go, but where I'm stuck is that it'd be way more
elegant to ditch this whole mess completely and remove headers from
the template.
It's too hard to write these headers by hand unless they are simple ones,
and there needs to be a way to pass in more complex headers that you want
to add. If I'm providing a way to pass in headers to @lamson.view.respond@
then why not take the headers out of the template and simplify the
whole rendering process?
I'd like to hear from people using Lamson or those who might have ideas on this.
If taking your headers out of the template and moving them to the render call
will totally screw up your day, then let me know. If it comes down to it, and I
have to do it, I'll probably write a conversion tool to help.
Also, if you have a suggestion for doing the above better but keeping the headers
then let me know.
Thanks for your help folks, and I'll be hanging out in the #lamson channel on irc.freenode.org
if people want to hash out a design.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-06-26.txt 0000644 0000765 0000024 00000007241 11221170627 023147 0 ustar zedshaw staff Title: Lamson 0.9.5, The Push To 1.0
I just released Lamson 0.9.5 with all the major Unicode refactoring done and working.
This is an important release because 0.9.5 is where I declare that I'm pushing to a 1.0
release and the base Lamson "APIs":/docs/api/ won't change under penalty of death. In fact they can't
change because I'm using them myself in a few applications.
From now on, if an API has to change, then I'll keep the old one around and attach a
deprecation warning, and then provide a new one. You'll then have by 1.0 to stop using
it and start using the new one. Except for a situations where a bug must be fixed, I
can't see the actual Lamson API needing such a drastic change by 1.0.
What will happen between now and 1.0 is bug fixing, and adding any nice additional
features people need within the current "API":/docs/api/ to help build applications.
This would be things like more connectors to other mail protocols (LMTP, QPMP), commands
for deployment features, and possibly a way to write your own command.
So, if you've been waiting for the API to stabilize then 0.9.5 is the release to grab
and start using. I'm currently using it on about 6 little projects and it's working
great so far, but I need more people to try it out and send me feedback on how well
it works for them.
h2. Getting This Release
As usual, you can read the "download":/download.html documentation to learn how to
grab it.
After you grab it, go read these documents:
* "Getting Started With Lamson":http://lamsonproject.org/docs/getting_started.html
* "A Painless Introduction To Finite State Machines":http://lamsonproject.org/docs/introduction_to_finite_state_machines.html
* "Deploying Lamson And OneShotBlog":http://lamsonproject.org/docs/deploying_lamson.html
Then you should hit the "API documentation":http://lamsonproject.org/docs/api/ and
start writing your application. Keep your first one small and try deploying it
in different ways.
h2. Simple Virtualhost Deployment Instructions Coming Soon
The "oneshotblog.com":http://oneshotblog.com/ sample application is now deployed in a
new virtualhost configuration on a new VPS I purchased this week. This configuration
is different from the ones given in the documentation above because it allows you
to run mulitple applications for multiple domains on a single machine.
I'll be writing up how to build your own version of this configuration today, but the
quick description is this:
# I have postfix running on port 25 now *in front* of Lamson.
# Postfix is configured to take all mail for each domain and dump it into a maildir queue for each one.
# The Lamson applications are then reconfigured in their @config/boot.py@ to use a "QueueReceiver":http://lamsonproject.org/docs/api/lamson.server.QueueReceiver-class.html to feed itself messages from its maildir.
# Lamson then just uses the same postfix server as its relay host like before.
The advantage of this configuration, apart from being able to easily host multiple domains,
is that you can shutdown your Lamson application and when you start it back up it
will catch up on the mail in the maildir. People will probably never know you did
maintenance or stopped the server temporarily.
The disadvantage of this configuration is that it's a royal pain to make postfix do this.
The instructions will show you step-by-step how to pull it off, so hopefully it'll make
it easier on people who need it. However, I'm gonna tell people to start small and
just use the simpler configuration "Lamson in front" configuration and leave this
for the big times.
h2. Mailing List Will Go Down
Finally, I'll be moving the mailing list shortly, so if you send a message and don't receive
a reply then that's why.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-07-03.txt 0000644 0000765 0000024 00000002330 11223705276 023143 0 ustar zedshaw staff Title: Article In The Reg About Lamson (By Ted Dziuba)
Just a quick update to point people at an article in "The Register":http://www.theregister.co.uk/ by
"Ted Dziuba":http://teddziuba.com/ entitled "Lamson - email app coding without the palm sweat: Doing what Java never did":http://www.theregister.co.uk/2009/07/03/lamson/. He interviewed me about Lamson, things I think you can use it for, and other fun stuffs.
A snippet from the article: "As a development platform, e-mail has gone
neglected for decades. Its esoteric implementation details and specifications
are regarded by many in the IT business as voodoo, best left to old-granddad
programs like Sendmail or Postfix. Zed Shaw hopes to change that with his new
project, Lamson. (The name Lamson is a throwback to the early 20th century
pneumatic tubes used to shuttle messages between offices. It was
originally called Son of Sam, but Zed's dog convinced him to change
it)."
"Read the article.":http://www.theregister.co.uk/2009/07/03/lamson/
h2. 0.9.6 Coming Soon
I've been cranking on various projects with Lamson and haven't ran into too many other bugs. I'll be
doing a code review of what's in the Bazaar repo and then pusing out a 0.9.6 probably this weekend.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-07-07.txt 0000644 0000765 0000024 00000000324 11224721637 023150 0 ustar zedshaw staff Title: Lamson Mailing Lists Down
I'm making up the release of Lamson and doing some server maintenance so I took down the
Lamson server running the mailing lists. I'll let everyone know when they're back up.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-07-09.txt 0000644 0000765 0000024 00000022024 11225570307 023150 0 ustar zedshaw staff Title: Lamson's Bounce Detection Algorithm
I just finished committing 0.9.6 code for doing bounce detection and analysis
with Lamson. It's part of the new mailing list example I'm coding up for the
0.9.6 release which I'll be running on a free mailing list site I'm going
to release soon. In this blog post I'd like to go through the bounce detection
algorithm and get some feedback and samples from people. So far it works great for
the samples I have, but I want it to be fairly bullet proof.
What I have here is a quick description of how Lamson is doing bounce detection,
and then how it gives you parsed information to do analysis beyond that. I'm hoping
that people who have to deal with bounces from systems like MS Exchange can possibly
either provide samples or advice so I can tune the algorithm better.
h2. The State of RFC 3464 And 1893
The two big RFCs for bounce handling are "RFC3464 - An Extensible Message Format for Delivery Status Notifications":http://www.faqs.org/rfcs/rfc3464.html
and "RFC1893 - Enhanced Mail System Status Codes":http://www.faqs.org/rfcs/rfc1893.html which
cover the headers and error messages you'll encounter.
In RFC3464 there's various headers that are given which a bounce message should have,
as well as the double and sometimes triple multipart-mime wrapping you need to do
to give back the bounced message. A major pain with this standard is that the headers
are spread out across the different nested parts, some might be duplicated, and the format
is *never* respected by the various MTAs out there.
With RFC1893 you have a large number of status codes, but they are combined in weird ways
such that you have a primary code with the first digit, a secondary code with the
second, and then a "third" code that's the second and third digit combined. Unlike HTTP
error codes where there's just a single number mapped to a single status code, in RFC1893
it's this strangely nested first+second+(second+third) thing.
Adding to this is there's a few headers where you can put these status codes, and in
various formats. Here's a sample of the *parsed* Diagnostic-Code from a gmail bounce
message:
'Diagnostic-Code': [(u'smtp',
u'550-5.1.1',
u"The email account that you tried to reach does...")],
The end result is that if you want to determine if a message has bounce (not even hard vs. soft)
then you have to go trolling through nested attachments looking for headers and parsing their values
on the chance they *might* be important to the bounce.
h2. Soft and Hard
The general consensus is that if a *primary status code* is greater than 4 then
the message is a *hard* bounce, anything else is a soft bounce. Of course there's no way
this is an absolute truth, and I'm sure there will end up being various complex
things people will want to do with a soft vs. a hard bounce.
Since there's no way Lamson could know what you want to base your decisions to honor a
soft vs. hard bounce, it simply lets you figure out which type it is, and then analyze
the information it's collected to determine the bounce.
h2. Lamson's Simplification
Lamson already converts the email into a normalized form, so the difficult part of
parsing out the bounce information is simply trolling through the parts and finding
all these randomly dispersed bounce headers. Then it's a matter of parsing the values
into something that can be use for analysis in code.
To simplify this, Lamson uses a dict of headers and matching regexes to search for in
the email:
Each of these headers shows up about once, maybe more in the parts of a message
that has bounced information, and they don't show up at all or rarely in regular
messages. What Lamson does is just walk through each part, finds the matches, parses
with the regex, and then produces something like this:
I've abbreviated some of the messages since they get stupidly long, but this shows you the information
you get now. Lamson also provides a nice API through the lamson.bounce.BounceAnalyzer class that
you'll see in a moment.
bq. If you know of additional headers and formats that show up in bounces, let me know.
h2. Bounce Probability
Now, we've collected up all the different headers you might find in a bounce package, and we've
attempted to parse the values into something meaningful. That means, the more of these headers
that we find and successfully parse, the more likely the message is a bounce.
Lamson keeps a score while it's finding the above headers, and for each one it finds and parses
it adds a "point". It then produces a probability <= 1.0 that the message is a bounce or not.
The more headers it finds, the closer to 1.0, the higher the chance. So far it looks like any
message above 0.3 is most likely a bounce message, with only a few spams that are below that.
Checking for a bounce then looks like this:
Which is from the test suite, and shows you various ways to get information
about the probability of the message being a bounce.
h2. Bounce Header Information
Finally, you'll need to get at this parsed information to make any
final decisions about the bounce. You might want to keep track of
which MTAs consistently fail and include them in your black lists.
You might want to tune your retries for different kind of soft
bounces.
Whatever you need to do, you have the original raw headers that
Lamson found, but you also have a higher level API you can use:
This demonstrates how I've pulled out all of the error codes mentioned in
RFC1893 and put them into Lamson so that you can get meaningful error
messages for the various multi-level error codes you'll encounter.
All of these error codes are available in the lamson.bounce module.
h2. Comments Or Suggestions
That's it for the review of Lamson's bounce detection. It's in the early
stages but already showing a lot of promise. It turned out to be much
easier than I anticipated, but only after doing the work in lamson.encoding
to clean up emails.
If you can think of situations that should be covered, and if you have samples
of horrible bounce messages, then I'd love to have them.
h2. Mailing Lists Coming Back
Tomorrow I'll be using this to reimplement the mailing list sample and put
it back into the bzr repository, then back online. I'll have a nice
status update about that tomorrow as I finish it up.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-07-14.txt 0000644 0000765 0000024 00000002006 11227105220 023130 0 ustar zedshaw staff Title: Lamson 0.9.6 Sometime Today
I think I'm at a point where bounce detection and handling works really well and is
ready for release. I've got it running on the latest Lamson demo "librelist.com":http://librelist.com/
and it works. I'll be pushing out a new release of Libre List in a little while
that does some advanced bounce handling, and if that all works, then I'll push
out 0.9.6.
This release will include bounce handling, the ability to change how state is
stored and maintained, the ability to route on the envelope rather than the
headers, and the full code to the librelist.com site including "Django Integration":http://dpaste.de/3qGB/
which really isn't "integration".
The plans for the 0.9.7 release are to remove all dependencies on SQLAlchemy to make
it clear that you don't have to use it, and any bug fixes. After that release, assuming
there's nothing major I need to add, I'll be doing the 1.0 pre-releases.
Shoot me email if you have questions, or hop on #lamson on irc.freenode.org to chat.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-07-19.txt 0000644 0000765 0000024 00000004462 11230540446 023154 0 ustar zedshaw staff Title: I Blame Bounces For The Delay
I was going to release Lamson 0.9.6 much earlier than this, but as I worked on
the bounce detection feature, I realized that if I just wrote that and a few
other cleanups and features, I'd actually have a *1.0pre1* candidate on my
hands instead of a 0.9.6 release.
Instead of releasing 0.9.6, I continued to work on "librelist.com":http://librelist.com/
and got the software smart as hell. While working on librelist.com I managed
to improve Lamson to the point where I'll be comfortable releasing a 1.0pre1
tomorrow.
h2. What's librelist.com?
If you go "librelist.com":http://librelist.com/ you can read about it, although
the site is still rough and being written. It actually does function however,
including basic archives.
For people too lazy to click on the link: librelist.com is just a free mailing
list service for open source projects, similar to freenode.org with IRC.
The primary purpose is to give open source projects a mailing list they can use that has:
* The code available (it's in the Lamson source).
* No logins, signups, bundled services, captchas, tracking cookies, or other crap getting in your way.
* No ads on your emails, ever.
* Full access to your archives using rsync.
* Easy and smart signup process.
* Spam blocking and bounce protection galore (although spam is off right now).
h2. Try The Lists
I'll have an announcement tomorrow laying out the big things you'll get, but for now
if you want to track Lamson and participate, as well as test the librelist.com
code, then try subscribing to "lamson@librelist.com":mailto:lamson@librelist.com
and saying hello.
If you're interested in librelist.com development, then subscribe to "meta@librelist.com":mailto:meta@librelist.com
and I'll be sending out announcements related to librelist there.
h2. The Lamson 1.0pre1 Plan
I haven't needed to touch Lamson much over the last few days, apart from some surgery
on the Python Maildir class to add a safety feature. The code currently in bzr is
also very stable and has been sped up a bit. I'm looking to basically spend tomorrow
auditing the code and writing docs with the goal of pushing out a release in the evening.
The goal then will be to make it solid and not add many more features so that I can get a 1.0
release out for real by the end of July.
See you tomorrow!
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-07-20.txt 0000644 0000765 0000024 00000007546 11230764565 023164 0 ustar zedshaw staff Title: Lamson 1.0pre1 Released
Tonight I'm releasing Lamson 1.0pre1 with all the latest improvements I've made while
making "librelist.com":http://librelist.com/ and taking feedback from a few people
using Lamson. The goal from now on will be to basically squash bugs and write docs, with
only a rare feature or two as I find them needed on projects.
h2. What's In This Release
This release features really only four major enhancements, and a bunch of little fixes
here and there.
# There's now a working mailing list example in examples/librelist.
# The router now uses the envelope header To (message.To) instead of the headers.
# There is now very solid bounce detection and analysis.
# Modules can specify anything as the state key (from, to+from, module+to, etc.)
Here's the important changes from the change log by category:
h3. Enhancements
* 340 Added a version command finally, with some good information.
* 330 There is now a safe=False option to lamson.queue.Queue to use a modified Maildir that md5 hashes the original hostname used.
* 321 Routing it more deterministic and a bit quicker. It now uses the envelope To for routing.
* 320 Refactored the routing to it is more deterministic and doesn't loop over the functions mulitple times. Still some work to do though.
* 314 Bounce information now includes the original messages in easier to access forms.
* 308 Got bounce handling working and tested.
* 303 Slight reorg with the way @state_key_generator is implemented.
* 302 Implemented a means to specify that a module will key off of different parts of a message.
* 301 Got tired of not seeing errors during test runs so I made an option in the Router to explode on exceptions vs. log them.
* 300 Bounce routing decorator for doing the actual routing of messages that are bounces.
* 296 Implemented a bounce analysis method that is simple but seems to work on most bounce mail.
h3. Bug Fixes
* 329 Found an email that had a nasty header causing an edge case in the parser.
* 325 Bounce detection was using the wrong email address.
* 319 Removed the egg dependency on sqlalchemy.
* 318 Moved the bounce stuff around in order to make it more advanced.
* 307 Fixed up the @bounce_to decorator so that it returns the next state.
* 294 Small bug in multipart handling where content types of 'message/' can be multipart too.
h3. Librelist Related
* 323 librelist.com sample is looking good, bounce handling is working, archives are started.
* 322 Super bounce handling for librelist.com.
* 306 Got the first cut of the whole app.handlers.admin working, including bounces, but Lamson bounce detection has a flaw.
* 305 Got the basics of making mailing lists, managing subscribers.
* 304 Initial commit of the librelist.com sample application.
Also, librelist demonstrates how to integrate Lamson with "Django":http://www.djangoproject.com/ which
turns out to just require a couple lines of code. I'll have some documentation on that soon.
h3. Docs and Website Related
* 317 Removed references to SQLAlchemy being needed in the docs.
h2. Getting This Release
As usual, try to install it using "easy_install":http://peak.telecommunity.com/DevCenter/EasyInstall
and make sure you do the upgrade:
$ sudo easy_install -U lamson
This release does not depend on SQLAlchemy anymore, since you can really use Lamson with any
data source you can access with Python. As mentioned above, the librelist sample uses
Django models.
If you want to download the source you can visit the "releases":/releases/ page and
grab it from there.
h2. Mailing List Back Online
Finally, the lamson mailing list is now back online, so you can subscribe to it
and get help. Read "the lists page":/lists/ to find out more.
h2. IRC Channel
I'm also more active in the IRC channel #lamson on irc.freenode.org. Come by
if you need help.
Thanks again folks, and shoot me any bugs you find.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-08-03.txt 0000644 0000765 0000024 00000035510 11235577723 023161 0 ustar zedshaw staff Title: Lamson 1.0pre2, HTML Email, Standalone
Lamson 1.0pre2 features two features that might signal the end of the beginning or
the beginning of the end, depending on your perspective: HTML Email and Lamson Standalone.
HTML Email support comes from a new module
"lamson.html":http://lamsonproject.org/docs/api/lamson.html-module.html that gives a nice template
method to send out HTML to victims...uh...customers. Lamson Standalone will be a way
to run Lamson as your customized email server instead of another server like Postfix.
Both of these features will be done for Lamson 1.0, but are currently just getting started
as of 1.0pre2. Once they're done that will be the last two requested features people
had before 1.0.
h2. Getting This Release
As usual, you can get this "release from /releases":/releases/ or if you are too lazy to
read that page then do this:
$ sudo easy_install -U lamson
In addition, if you want play with the new features you'll need to install these (optional)
Python libraries:
I'll decide if I should make these mandatory or optional with the next release based on
people's feedback.
h2. Lamson Standalone
Quite a few people have asked for a way to install Lamson as their primary email server.
Usually they want it for just their own email on a machine just for them where installing
something like Postfix is overkill or just too hard. This is entirely possible
with Lamson, but it's kind of not the right use for it since simple delivery of mail is
much better handled and implemented by an older server like Postfix.
Rather than resist giving people a way to setup Lamson as their primary mail server, I've
decided to start making it possible for them. By the next release (1.0pre3) you'll be able
to run Lamson as your only email server for small installations handling mail for one person.
I'll also include a "screen cast":/videos/ showing people how to use the standalone functionality
to build a *Personal Mail Management Server* to filter and control their mail without any
other mail server. I'm going to treat this as a first project for most people wanting to get
started using Lamson for their email.
As of 1.0pre2, the functionality that's available is code in the "lamson.server.Relay":http://lamsonproject.org/docs/api/lamson.server.Relay-class.html
that will use "PyDNS":http://pydns.sourceforge.net/ to query up the MX host for
recipient addresses. It's fairly primitive right now, but you use it by creating
a Relay with the host explicitly None:
relay = server.Relay(host=None, port=25, debug=1)
Once you do this the Relay will lookup the hosts rather than trying to send through a relay
host.
Don't go crazy with this yet, since it has to be tested with various kind of nasty email
addressing out there, and it needs to have a way to generate a bounce when it has an error.
h2. HTML Email
Well, I broke down and implemented my idea for making HTML Email easy as hell to generate.
In the past I didn't want to include simple HTML support because, well, HTML Email is
annoying as hell and I didn't want to deal with the support headaches. I knew once I threw
HTML generation into Lamson I'd have an army of marketing people using Lamson poorly to
generate their marketing materials. I also think that HTML formatting in email doesn't work
as a customer development strategy.
Yet, every time I tell someone about Lamson, the very first, second, third, and 300th thing they
ask is if it does HTML Email. Over and over and over this was the most important feature, above
spam blocking, filtering, building applications, intelligent state management, or anything else
that Lamson supports.
Well, if the people want HTML email generation, then the people will get it. I introduce
you to "lamson.html":http://lamsonproject.org/docs/api/lamson.html-module.html which
makes it trivial to produce HTML in your email and doing it in a nice clean way using
"CleverCSS":http://sandbox.pocoo.org/clevercss/ and "Jinja2":http://jinja.pocoo.org/2/
templates.
Let's start with the simplest little example that will send out a disgusting html template:
import sys
from lamson import html, server
from config import testing
relay = server.Relay(host=None, port=25, debug=1)
hs = html.HtmlMail("style.css", "html_test.html")
title = "Test Message HTML"
msg = hs.respond(locals(), "content.markdown", From=sys.argv[1],
To=sys.argv[2], Subject="Test %(title)s")
relay.deliver(msg)
You could run this right out of the Lamson source tree like this:
The result would look like this in victim's email:
!html_email_in_gmail.png!
h2. How lamson.html.HtmlMail Works
The rationale behind the HtmlMail class is that you'll typically have a template you
want to send, some CSS, and then body content that you'll plug into the template
as a slug for each person. What HtmlMail does is let you setup the template as HTML
with Jinja2, and then specify your CSS using CleverCSS (which is way easier).
When you go to send, you generate a "Markdown":http://daringfireball.net/projects/markdown/
template as the body. This lets you write the actual body of your HTML mail the same
way you would a regular email, but still let's you get a good HTML output using the
HTML/CSS templates. With markdown regular folks can write the marketing copy for
your spam while keeping the design separated.
HtmlMail then glues this all together by doing the following:
# Parses your CleverCSS template (after running it through Jinja2).
# Runs your content markdown through Jinja2 and "markdown2":http://code.google.com/p/python-markdown2/ just like all your other Lamson templates.
# Injects your CSS *into* your HTML tags so that you get your styles even though many clients rip out the *style* tags from your HTML. See the output sample below.
# Knits your generated content into your HTML template as the {{content}} variable.
The end result is that these two lines of code from the above sample:
Will let you blast out well formatted HTML emails that have a high probability
of displaying in most mail clients.
h2. How The HTML And CSS Looks
Here's what each of the files in the above sample look like. First the @style.css@:
body:
margin: 10
padding: 20
background: green - 30
color: blue
h1:
font-size: 3em
h2:
font-size: 2em
color: yellow
h2:
font-size: 1em
p:
padding: 0.3em
background: red
h2:
color: yellow
#bright:
background: black
color: white
.dull:
background: gray
color: black
Notice that "CleverCSS":http://sandbox.pocoo.org/clevercss/ supports quite a few
very cool features, like calling functions, nesting, variables, and calculations.
Next you have the outer template @html_template.html@ that wraps your content markdown template. Notice
that there's no *style* tag where the above CleverCSS is placed. That gets done
later by lamson.html.HtmlMail.
<html>
<head>
<title>{{ title }}</title>
</head>
<body style="background: magenta">
<h1 class="bright">{{ title }}</h1>
{{ content }}
<h2 id="dull">All done.</h2>
</body>
</html>
Finally, you have the @content.markdown@ file that has your basic markdown formatted email:
Hello
=====
I would *love* for you to tell me what is going on here joe. NOW!
Alright
-------
This is the best I can come up with.
Zed
You can use any format you want by changing a setting when you construct your HtmlMail object,
but markdown seems to be the closest to what people would send as plain text for their email.
In fact, I'd like to find a way to send the raw markdown as the plaintext version of each HTML
email that goes out.
With the above three components, you then get the following output:
<html>
<head>
<title>Test Message HTML</title>
</head>
<body style="margin: 10; padding: 20; background: #006200; color: blue">
<h1 class="bright" style="background: black; color: white">Test Message HTML</h1>
<h1 style="font-size: 3em">Hello</h1>
<p style="padding: 0.3em; background: red">I would <em>love</em> for you to tell me what is going on here joe. NOW!</p>
<h2 style="color: yellow">Alright</h2>
<p style="padding: 0.3em; background: red">This is the best I can come up with.</p>
<p style="padding: 0.3em; background: red">Zed</p>
<h2 id="dull" style="background: gray; color: black">All done.</h2>
</body>
</html>
Like magic HtmlMail has taken your CleverCSS file and merged it into the tags of your HTML to
style it, making it work in most clients without forcing you to do it manually. It does this by
walking the CSS and generated HTML with "BeautifulSoup":http://www.crummy.com/software/BeautifulSoup/
and setting the *style* attribute as it goes.
h2. Librelist JSON Archives
There's been a ton of work on "librelist.com":http://librelist.com/ to make it
work better, but the big feature is a complete JSON dump of the mail archives
and a new "Archive Browser":http://librelist.com/browser/ for reading the
archives.
The browser is just a fast one day hack I did to prove that the JSON
archive format was good. It uses "JQuery UI":http://jqueryui.com/ to build
the interface, and has *no servers side software other than Nginx.* Yes, you
read that right, there is no Django behind this, just a pure "Nginx":http://nginx.net/
setup serving files and directory indexes.
The real meat of this setup is the Lamson server on librelist and an Nginx module.
First, the librelist server generates a JSON version of each archived email which
you "can see here":http://librelist.com/archives/lamson/2009/07/30/json/1248947932.M724307P15430Q6.09c5769d5b9f3d575cefc2ccb51877ec.json with your browser. This JSON actually loads as an
object in JavaScript, so you have code like this:
Second, there's a tiny modification to Nginx's
"autoindex":http://wiki.nginx.org/NginxHttpAutoindexModule I made so that
it will generate a "JSON version of the index instead of HTML":http://librelist.com/archives/lamson/2009/07/
so that you can "browse" the stored JSON files via JavaScript:
function loadDay(date) {
if(window.MAILING_LIST) {
$.getJSON("/archives/" + window.MAILING_LIST + date + "/json/", {},
function(data) {
$("#messages").empty();
base = data[0]
msgs = data[1]
for(i in msgs) {
fetchMessage(base + msgs[i])
}
});
} else {
$("#messages").html("<h2>Now pick a list to look at for that day.</h2>");
}
}
With those two components I can export a JSON version of the mailing list archives without
having to run a large web framework.
h2. ChangeLog
For those who are interested, here's the Bazaar logs from this release, separated into
changes to Lamson and changes to the included librelist.com sample.
h2. Lamson Changes
* Cleanup and tuning of the html code.
* Documenting the HtmlMail API.
* New HTML email generation with CleverCSS support and merging the CSS into the HTML results.
* Additional tests for the new MX query style of Relay.
* Implemented a feature to let the relay do its own delivery, rather than needing a smart host. Requires PyDNS.
* Cleaned up the build so that it removes junk from the examples.
* Fixed up the lamson default help command so that it is more succinct.
* Properly skips spam filtering if the spam databse doesn't exist.
* Made the spam filter not barf if there's no spam db, which helps when testing.
* Now don't bother storing the START state since it's the default.
* Cleaned up mail some more and deprecated the MailRequest.msg and MailResponse.msg to be replaced with .base (since that's what it is).
* Added a simple method to copy the attachments of a MailRequest to a MailResponse.
* Got decent attachment copying implemented, mail api needs a bit of cleanup.
* Return an empty string when trying to encode a none.
* Minor bug in the safequeue, added logging so you can track oversize.
* Getting the headers right for replies to work, removed the first message.
* Added options to Queue and QueueReceiver to limit the size of incoming mail.
* Added a version command finally, with some good information.
h2. librelist.com Changes
* Force the permissions to be correct for serving up the archives.
* Make sure that bounces are always saved for later.
* Json convert script can't go there.
* Need to install simplejson to make the json work on librelist.
* Prevent loop back of messages from lamson to itself, and stop messages that should not be considered lists like unbounce and noreply.
* Implements a json version of the mailing list archives in addition to the regular MIME encoded ones.
* Update of changes from the live server for setting up the spam system.
* Migration to install the spam database on deployment.
* Added spam filtering to the admin handler where it counts.
* Confirmations are removed after they are verified.
* Template for bad list name help.
* Tweaked the formatting for the bad list name helper, and made sure that it returns to START.
* Now giving out an error message for people who get the name format wrong.
* Added additional headers needed for achives, and try to maintain the date and message-id headers.
* Production and staging deployment scripts setup and a migration to make those work.
* Implemented a simple bash based deployment setup for automating librelist deploys and migrations.
* Nope, can't add the - yet without some regex wizardry.
* Small change to allow - in the mailing list names.
* Updated the messages librelist sends out for readability.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-08-22.txt 0000644 0000765 0000024 00000012217 11243717140 023145 0 ustar zedshaw staff Title: Lamson 1.0pre4 Out, Lots Of Docs Done, 100% Coverage
I happy to announce probably one of the last few releases before
I officially put the 1.0 stamp on Lamson. This last 1% of the things
I want to do takes a while, but it really puts a good shine on the
project.
What I've got in 1.0pre4 is "tons of docs":http://lamsonproject.org/docs/
and 100% test coverage. Most of the features that were added are
nice-to-haves that I've found useful while developing my various sites.
h2. Getting The Release
As usual, @sudo easy_install lamson@ is your friend. If you want
to use the HTML generation features, then you'll want to also install:
BeautifulSoup
CleverCSS
markdown2
For completeness, here's the remaining packages I have installed in most
of the virtualenvs for my applications:
You can also grab "source releases":/releases/ and "download instructions":/download.html
from this site if PyPI is failing you.
h2. 100% Test Coverage
First, I've managed to get the test coverage for Lamson up to 100%:
It's mostly a vanity thing, but doing this did uncover a couple of
minor little bugs here and there, and it makes people feel better
trusting Lamson.
h2. Docs, Docs, Docs
I firmly believe that a good software project has *both* "API":http://lamsonproject.org/docs/api/lamson-module.html
style documentation and guided documentation. That's why I spent almost this whole
time working on documenting all the parts of Lamson people need and making sure the generated
docs were complete.
What I've done is reorganized the documentation section so that you can find
topic by their categories, and then documented most of the features people
need to use daily. Here's some highlights:
* "Unit Testing":http://lamsonproject.org/docs/unit_testing.html
* "Confirmations":http://lamsonproject.org/docs/confirmations.html
* "Filtering Spam":http://lamsonproject.org/docs/filtering_spam.html
* "Bounce Detection":http://lamsonproject.org/docs/bounce_detection.html
* "HTML Email Generation":http://lamsonproject.org/docs/html_email_generation.html
* "Unicode Encoding/Decoding":http://lamsonproject.org/docs/unicode_encoding_and_decoding.html
* "Hooking Into Django":http://lamsonproject.org/docs/hooking_into_django.html
* "Tons more...":http://lamsonproject.org/docs/
I especially like how the HTML Email generation came out as a feature, but I really
like the "Bounce Detection":http://lamsonproject.org/docs/bounce_detection.html
documentation the most.
h2. New Confirmation API
This release has a few little bug fixes, but mostly it has a solidified feature for
doing "Confirmations":http://lamsonproject.org/docs/confirmations.html easily.
h2. HTML/Text Dual Email
Previously the "HTML Email Generation":http://lamsonproject.org/docs/html_email_generation.html
feature could only generate HTML email reliably, but I managed to track down
a bug that lets you now craft dual email. I even went a step further and made it easy
to simply use your markdown content templates as your text/plain alternative
when you send out HTML Email.
h2. Multiple Recipient Routing
There was no way to avoid this, so I just implemented it. Now when a message is for
multiple recipients (in the envelope) Lamson will route the message to each
address. That means if it is destined for multiple people, then Lamson will
effectively try to match each of those people and run them through your handlers
separately.
I was worried this would cause problems, but so far it's working fine. If you see
that it can be used as a DOS attack, then I'll work on a way to limit the number
of recipients that Lamson will allow (similar to how you can restrict the size
of messages off a Queue).
h2. Test And Report
As usual, please test this out, review the documentation, and report back to me
any problems you find.
I'll be in #lamson on irc.freenode.org more, and will probably install a IRC
bot to notify when I post new code to the repository.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-09-07.txt 0000644 0000765 0000024 00000002417 11251323145 023147 0 ustar zedshaw staff Title: Lamson 1.0pre5 Out
I just pushed Lamson 1.0pre5 to PyPI for your enjoyment:
"PyPI Download":http://pypi.python.org/pypi?:action=files&name=lamson&version=1.0pre5
You can also get it from "releases":/releases/ or by doing the usual
easy_install.
This is a small bugfix release that only adds one small feature. If
you're running with the SMTPReceiver, you can raise a server.SMTPError with
an error code you want and the SMTPReceiver will return that as the error
code to the client.
For example, if you do this in your handler:
Then it will generate a reasonable error message (using lamson.bounce
messages) and give that to the client. You can also give a second
parameter to SMTPError that will be the error message if you don't
like the automatic ones.
Only warning with this is use it sparingly, as it then ties you always
running Lamson as an SMTPReceiver, instead of off the QueueReceiver. It also
leaks out to malicious clients more information about your server than
you might want them to know. In general, it's better to just ignore bad
messages and save them in an undeliverable queue for later inspection.
Alright, try it out and report bugs to me. Thanks.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-09-26.txt 0000644 0000765 0000024 00000002264 11257445456 023170 0 ustar zedshaw staff Title: Quick Lamson 1.0pre6 Release, More This Weekend
I put up a quick release of Lamson that fixes a bug in the gen project prototype.
I recently switched all the domain names being used in the test suite to "localhost" or
a similar nonexistant domain name. Why? Well, because some people liked to run
the Lamson test suite or their fresh project test suite against a *live* SMTP
server. The test suite would "send" mail to test@test.com and my own
address, under the assumption that people would use the @lamson log@ test server.
End result: Periodically I'd get bounce messages from test.com and other test
suite mail.
Well, I made that change, but forgot to update a couple places in the prototype,
so newbies were seeing the tests for their fresh project fail and wondering
if Lamson was broken.
Sorry about that.
You can grab this 1.0pre6 release from the "releases":/releases/ area, and
you can also do the usual @easy_install -U lamson@ to get it.
I'll be doing some more work to fix a few bugs this weekend and working on a
fun new site for guitarists, so stay tuned. And if there's something in particular
you want in Lamson just email the lamson@librelist.com mailing list.
lamson-1.0pre11/doc/lamsonproject.org/input/blog/2009-12-12.txt 0000644 0000765 0000024 00000007011 11311017422 023123 0 ustar zedshaw staff Title: Lamson 1.0pre9 Is Out
*UPDATE*: This used to be an announce for 1.0pre8, but python-daemon apparently has a bug that makes it not honor
the TERM signal (still) so I had to go back to HUP.
I just got done knocking down all the bugs in the new "Fossil":http://fossil-scm.org/ based
"Lamson Support Site":http://support.lamsonproject.org/ and cooked up 1.0pre9. This has a
ton of fixes, but I'll let the ChangeLog speak for itself:
=== 2009-12-12 ===
22:49:43 [c2be9dedc3] *CURRENT* Version bump to 1.0pre9 thanks to python-
daemon. (user: zedshaw tags: trunk, 1.0pre9)
22:43:04 [1718d014a5] Stupid fucking python-daemon doesn't recognize TERM yet.
(user: zedshaw tags: trunk)
20:24:20 [b5796e14a1] Pointed at the tag instead of just the leaves. (user:
zedshaw tags: trunk)
20:00:38 [42768ade2b] 1.0pre8 release announced and done. (user: zedshaw tags:
trunk, 1.0pre8)
19:43:53 [bb1b0a0dae] Updated MailBase.__nonzero__ to include parts as well.
(user: zedshaw tags: trunk)
19:40:51 [2916b83307] Created ChangeLog from fossil timeline of commits.
(user: zedshaw tags: trunk)
19:36:14 [41530d8a34] Version bump to 1.0pre8 for release. (user: zedshaw
tags: trunk)
19:31:21 [dfe158b282] Initial support of using Delivered-To or similar more
exact To header. (user: zedshaw tags: trunk)
19:14:13 [e9988679b7] Updated prototype.zip for upcoming release. (user:
zedshaw tags: trunk)
19:13:51 [98e94b6103] Potentially fixed the unicode/ascii issues with shelve
storage of confirmations and routing. (user: zedshaw tags: trunk)
18:56:35 [6224ae68e4] Updated docs about confirmation to fix a doc bug. (user:
zedshaw tags: trunk)
18:56:14 [103454c43c] Make things clean up better on INT or TERM (not HUP).
(user: zedshaw tags: trunk)
18:25:29 [009451cabc] Added meta tags to the site so it can be found easier. I
didn't know search engines still use that crap. (user: zedshaw tags:
trunk)
18:22:24 [8908d7852d] Applied starttls, username, password patch to server.
(user: zedshaw tags: trunk)
18:16:01 [9a9f9f37b6] Applied patch for handling odd email addresses and white-
listing what headers have email addresses. (user: zedshaw tags: trunk)
18:06:06 [7605ef0517] Patch for queue bugs applied [bug 9d330c8f80] (user:
zedshaw tags: trunk)
18:02:50 [ef9808c4f7] Point at the right archive browser. (user: zedshaw tags:
trunk)
18:01:39 [3b3dc9391a] Better logging message on failure to connect to relay.
(user: zedshaw tags: trunk)
=== 2009-12-05 ===
18:05:40 [ddc4369b1d] Initial commit to the new fossil based support site.
(user: zedshaw tags: trunk)
17:48:22 [8140fcad03] initial empty check-in (user: lamson tags: trunk)
lamson-1.0pre11/doc/lamsonproject.org/input/contact.txt 0000644 0000765 0000024 00000000640 11204533220 022602 0 ustar zedshaw staff From: Zed
Title: Getting Help With Lamson
There are "mailing lists":/lists/ available (running Lamson of course) that you can join
to get help. Keep in mind that these lists are running Lamson, and since Lamson is still a
work in progress they might be faulty.
The best way to get help with Lamson is to email me directly. You can send
mail to "zedshaw@zedshaw.com":mailto:zedshaw@zedshaw.com to ask for help.
lamson-1.0pre11/doc/lamsonproject.org/input/docs/ 0000755 0000765 0000024 00000000000 11313464573 021355 5 ustar zedshaw staff lamson-1.0pre11/doc/lamsonproject.org/input/docs/bounce_detection.txt 0000644 0000765 0000024 00000043554 11243654672 025445 0 ustar zedshaw staff Title: Bounce Detection
Lamson supports intelligent bounce detection with its
"lamson.bounce":http://lamsonproject.org/docs/api/lamson.bounce-module.html
module. The bounce handling is based on a probability that, depending on found
headers, the message is a bounce. It then gives you a nice clean interface to
check who it's from, the originating SMTP server, the error messages, and any
human readable messages.
h1. How Bounces Actually Work
Figuring out how a bounce is actually handled is a bit difficult because the
majority of the information available is written by people who know very little
of SMTP server operations. When the average programmer thinks of handling a
bounce, she usually has one of these concepts in mind:
# The message could not be delivered, so the remote SMTP server sent back a bounce message.
# The message could not be delivered, so the local SMTP server sent back a bounce message.
# The message could not be delivered, so the recipient's email client sent back a bounce message.
# The message could not be delivered, so Lamson crafted a bounce message.
Obviously logically you can't have the recipient's email client send you a
bounce unless the user does something weird (and incredibly annoying). This
makes sense because the message was @not delivered@. How can the email client
send back a bounce if they don't receive the message.
bq. Yes, some clients do support sending bounces, but very few people use this
feature. If you do please talk about it as one more edge case to deal with.
Next Lamson can't craft the bounce messages for you because Lamson is simply
trying to deliver outgoing mail to a smart-host. Lamson does nothing but ask
the smart-host (your local SMTP server) to deliver, and then waits for a
response. That means Lamson is not sending you any kind of bounce unless you
write code to make it do that.
That leaves only two options for @who@ is really sending the bounce message:
the remote MTA or the local MTA. The truth is it's a little bit of both.
h2. Your Local MTA Is A Person
How a bounce works in practice involves two SMTP servers: your local
smart-host, and the remote server that it tries to connect to for delivery.
The message you actually get in lamson is from the @local@ server, and usually
an address @at@ that local server. It does not come from the remote server,
but inside your bounce will be a message and status indicators from the remote
server indicating why it bounced.
What happens is your local server attempts to deliver, and the remote server
rejects it for some reason. Now your local server is supposed to try again in
certain situations, but after a certain number of rejections or failures it
crafts a bounce message. That bounce message is then returned to your @lamson@
server based on the @envelope@ header of the message (more on that later).
Now, what's inside this message? Well, it's a oddly nested barely standardized
pile of random other messages. This is the frustration with bounce handling.
You pretty much either have to use a probability mechanism (like Lamson does)
or you have to craft your bounce handling for your very specific local server
and any other possible servers you talk with. Even then you can have problems
dealing with bounces reliably.
Inside this description is an important concept to understand:
bq. Lamson does NOT process bounces from the remote MTA or the remote user in
any way. Lamson process bounces from the @local@ SMTP server.
This is important because if you try to use the bounce message as if it comes
from the remote user, then you'll accidentally put your @local@ server into a
state that prevents you from processing future bounce messages.
If you did not get that, reread this whole section again until moving forward.
h2. "Hard" vs. "Soft"
Another complexity in dealing with bounces is the concept of "Hard" vs. "Soft"
bounces. My opinion is that the distinction is fairly retarded since it is
almost entirely unreliable and has no meaning to someone trying to handle a
bounce.
In popular terminology the main difference is this:
* Hard bounce means that person does not exist, so I can not sell him any more crap and need to remove him (maybe).
* Soft bounce means that person still exists, but my marketing bullshit didn't get through, I should try again 10 or 15 more times until he gets my important message about winning the lottery.
You may be laughing, or screaming various pedantic specifics, but the above two
distinctions are both how many email services look at bounces, and how most
malicious mail users view them.
For the mail services, leaking a hard or soft bounce is a security problem
because it tells a malicious sender if that address is a valid person, why the
sender was blocked, and how they can work around it. This is why many of the
error messages you get back are vague and mostly lies. The major email
services just don't want to give you any information that you can use to
circumvent their anti-spam measures.
bq. The services that have the strictest anti-spam measures also have the most
nasty disgusting spam on the web pages displaying user's mail. Yahoo is both
the worst for erroneously blocking email and for showing the worst most tricky
spam all over every square inch of their mail service.
How does this relate to your Lamson application? It basically says that you
should treat bounces as being basically soft bounces all the time, and then
tune your rules heuristically over time as you run into more and more.
h2. VERP
The final topic to touch on before getting into how Lamson handles bounces is
one of VERP. Remember in the description of bounce handling above I said that
your @local@ MTA sends the bounce message back to the @envelope from@ not the
From in the headers (well, most ones will). Because of this you can have a
From in the envelope that is only replied to when there's a bounce, and then
put the real From in the headers for normal processing.
This is effectively what "Variable Envelope Return
Path":http://en.wikipedia.org/wiki/Variable_envelope_return_path does to
process bounce messages. Rather than attempt to parse the body of a bounce
message, VERP rewrite the From address so that when a bounce is returned, you
only have to process the returned address to see what the original was.
Lamson could potentially support this, but it has several problems for generic
bounce handling which meant that actually parsing bounce bodies was a better
option.
h2. Using Lamson's Bounce Handling
Using Lamson's bounce handling is fairly simple. It involves the following
process:
# Import a special decorator @bounce_to@ from "lamson.bounce":http://lamsonproject.org/docs/api/lamson.bounce-module.html
# Create two (or one) handlers to deal with bounces.
# Place the decorator on any @START@ entry points to your handlers that can receive bounces, pointing them at your handlers.
# Handle the bounces in your handlers, making sure to return back to the @START@ state for the local MTA (remember, the local MTA is a person for bounce handling).
Here's some simple code that does exactly this by just ignoring bounces from
the "myinboxisnota.tv":http://myinboxisnota.tv/ example:
This example is stripped down from the real code so you can see what's going
on. If we walk through this you can see what happens:
# First we import the @BOUNCES@ variable from @config.settings@ which is just the queue we want to dump bounces into.
# We then create a special handler named @IGNORE_BOUNCE@ that accepts a message to anything in its @route@ and just puts the message in the @BOUNCES@ queue.
# This @IGNORE_BOUNCE@ handler then immediately returns @START@ so we go back to the START state.
# On the @START@ state you'll see that we have our @bounce_to@ declaration that points for @hard@ and @soft@ bounces at @IGNORE_BOUNCE@.
# This decorator wraps your handler in a little check that, if the message is a bounce, your @START@ state won't get called, and instead your @IGNORE_BOUNCE@ state will.
That is pretty much all you need to deal with to re-route bounces somewhere else.
You can also redirect them to a totally different handler, which is exactly
what the "librelist.com":http://librelist.com/ example does.
h2. How It Works
How does Lamson figure out that something is a bounce? What Lamson does is it assumes bounces will have some or all of these
headers:
The problem traditionally with parsing a bounce message was that, while you
need to find all of these headers, there was no real standard as to how the
messages in the bounce message are structured. From my
"postfix":http://www.postfix.org/ server the bounce message is a sequence of
about 6 nested attachments containing other messages, and sometimes the nesting
goes three deep.
Rather than rely on this structure (which changes all the time) or that these
headers are always present (they aren't), Lamson takes a probabilistic approach
based on the number of headers and properly formatted values it finds in @all@
nested attachments. The process goes something like this:
# Traverse all the possible nested attachments.
# Try to find each header in the attachment. If it's found add a point.
# If the header is found, use the regex associated with it (above) to try to match the value.
## If the value matches, then keep the regex captures for later. Add another point.
# For each header found, and any regex captures that matched the bodies, put them into an internal dict for analyzing the bounce information.
# Finally, calculate a probability score that is the total number of BOUNCE_MATCHERS * 2.0 / points.
In general, if a message is found that has a 0.3 or higher bounce probability
then it is considered a bounce message and you can look at it. The @bounce_to@
decorator has a threshold you can adjust if you want to be more or less strict.
The final result of this processing (which is actually very fast) is that any
calls to @MailRequest.is_bounce@ will either return True or False, and then
after you call is_bounce you can access the @MailRequest.bounce@ attribute to
analyze the information. That information is captured and cooked into a
"BounceAnalyzer":http://lamsonproject.org/docs/api/lamson.bounce.BounceAnalyzer-class.html
object.
h2. What It Looks Like To Receive One
It's also instructive to see what it looks like when Lamson processes a bounce
message. Here's the "librelist.com":http://librelist.com/ server processing
a bounce message:
2009-08-21 13:43:47,223 - root - DEBUG - Pulled message with key:
'1250876622.V8e00I219de0M128371' off
2009-08-21 13:43:47,231 - root - DEBUG - Message received from Peer:
'/var/mail/vhosts/librelist.com/delivery/', From: u'"SPAMMER"
', to To [u'lamson@librelist.com'].
2009-08-21 13:43:47,251 - routing - DEBUG - Matched u'lamson@librelist.com'
against START.
2009-08-21 13:43:47,332 - routing - DEBUG - Message to
set([u'lamson@librelist.com']) was handled by app.handlers.admin.START
2009-08-21 13:43:57,367 - root - DEBUG - Pulled message with key:
'1250876627.V8e00I219661M719350' off
2009-08-21 13:43:57,381 - root - DEBUG - Message received from Peer:
'/var/mail/vhosts/librelist.com/delivery/', From:
u'MAILER-DAEMON@librelist.com (Mail Delivery System)', to To
[u'lamson-confirm-74e2ca94b24a4be18da277f4666a6494@librelist.com'].
2009-08-21 13:43:57,410 - routing - DEBUG - Matched
u'lamson-confirm-74e2ca94b24a4be18da277f4666a6494@librelist.com' against START.
2009-08-21 13:43:57,431 - routing - DEBUG - Message to
set([u'lamson-confirm-74e2ca94b24a4be18da277f4666a6494@librelist.com']) was
handled by app.handlers.admin.START
These log messages show the following interaction:
# SPAMMER@hotmail.com tried to spam the lamson@librelist.com mailing list.
# They were required to subscribe, so Lamson sent them a confirmation mail.
# That message bounced, so Postfix sent back a bounce message from MAILER-DAEMON@librelist.com to Lamson.
# This message from MAILER-DAEMON is a bounce, so the librelist code handled it on the START state, NOT the CONFIRMING_SUBSCRIBE state.
# Internally, librelist looked up the target user and just zapped them.
That shows how the bounce doesn't come from SPAMMER@hotmail.com nor any server
at hotmail.com, but instead from MAILER-DAEMON@librelist.com. You could also
get messages from a remote MTA, but if they were found to be bounce messages
then that remote MTA would be treated like your own MTA.
h2. Gettting Bounce Information From BounceAnalyzer
The "BounceAnalyzer":http://lamsonproject.org/docs/api/lamson.bounce.BounceAnalyzer-class.html
does the work of figuring out additional useful information you can use to
determine what to do with the bounce. It looks at the final headers that are
scanned in the above process and pulls out important information everyone
needs. The list of information you can get is:
* primary_status -- The main status number that determines hard vs soft.
* secondary_status -- Advice status.
* combined_status -- the 2nd and 3rd number combined gives more detail.
* remote_mta -- The MTA that you sent mail to and aborted.
* reporting_mta -- The MTA that was sending the mail and has to report to you.
* diagnostic_codes -- Human readable codes usually with info from the provider.
* action -- Usually 'failed', and turns out to be not too useful.
* content_parts -- All the attachments found as a hash keyed by the type.
* original -- The original message, if it's found.
* report -- All report elements, as lamson.encoding.MailBase raw messages.
* notification -- Usually the detailed reason you bounced.
But, refer to the documentation for more accurate listings. An important
feature is that the status codes are parsed and converted into a standard list
available in @lamson.bounce@ based on their numeric values. Rather than parse
the details given by the remote MTA, you just use the number codes to get a
human readable output.
The best way to see all that's possible is to take a glance at the Lamson unit
test for the BounceAnalyzer:
Here you can see that you can figure out if the bounce is hard vs. soft, get a
human description, access status codes of various flavors, get the remote MTA's
name, the reporting MTA (your local), who it was originally for
(final_recipient), and even access the raw @bounce.headers@ if that's not good
enough.
h2. Augmenting The Matchers
Another advantage of this method of processing the bounces is that if your SMTP
server crafts something that hasn't been handled, then you can augment the
matchers being used. Simply update the @lamson.bounce.BOUNCE_MATCHERS@ dict
with your new ones and make sure to update @lamson.bounce.BOUNCE_MAX@ to be 2
times that.
The status codes are also available in the same way. Refer to the source for
more information.
One tricky part of Lamson's bounce handling is that it does assume a certain
structure for the BounceAnalyzer to get at the internal details. This
structure is the one used by Postfix, and it should be the same for other
servers. However, if you run into a structural difference, report the results
back so the handling can be improved.
h2. A More Complete Example
Finally, the "librelist.com":http://librelist.com/ example code has a much more
complete example of using bounces to disable users and shift their state.
Rather than describe it in detail here, I'll simply point you at the "source
releases":/releases/ so you can see it for yourself. Look in
@examples/librelist/app/handlers/bounce.py@ to see how it all works.
In fact, studying how this is triggered from the rest of the librelist example
is a great way to learn how to use Lamson in an advanced fashion. Study well.
h2. Conclusion
Lamson bounce handling is very advanced and can deal with a wide range of
scenarios. It should work with a wide range of bounce styles and other
servers, but feel free to report your own experiences and differences.
lamson-1.0pre11/doc/lamsonproject.org/input/docs/confirmations.txt 0000644 0000765 0000024 00000030576 11310760160 024770 0 ustar zedshaw staff Title: Confirmations
You never know who's going to send an email to your Lamson application. It may
be a real person, or a spam bot. Currently most of the spam bots aren't very
intelligent (probably because they haven't discovered Lamson yet) so a simple
easy way to weed out the randomness is to confirm people at important "choke
points" in your application.
Lamson provides a simple API in
"lamson.confirm":http://lamsonproject.org/docs/api/lamson.confirm-module.html
to help you send out and verify confirmation emails. You provide a storage
class for keeping track of who you're expecting to confirm, a template for your
message to send, and it does the rest. It's even advanced enough to keep the
original email around so you can resend it after the confirmation.
h2. The General Theory
The confirmation API assumes a few things about how you're doing your
confirmations:
# You want them to reply to an email to confirm.
# The address they reply to confirm has a "target" to differentiate it from other parts of your application that needs confirmation.
# The address they reply to also has a hex formatted randomly generated UUID for security, and your storage can handle that length.
# You want to store the message they sent so you can get it back after they confirm.
With those fairly reasonable assumptions you only need to then setup the
confirmation API in your @config/settings.py@ file and use it in your handlers,
preferrably at your @START@ state and anywhere else that you need
to validate the user before they do something destructive.
h2. Simplest Usage
The most basic usage of the
"lamson.confirm":http://lamsonproject.org/docs/api/lamson.confirm-module.html
API is to put it in your @config/settings.py@ and then access it in your
handlers. You configure it like this:
from lamson import confirm
...
CONFIRM_STORAGE=confirm.ConfirmationStorage()
CONFIRM = confirm.ConfirmationEngine('run/pending')
This puts a variable in your settings.py that you can access from your handlers
to craft and verify the confirmation messages.
bq. By default uses a simple in-memory dict to store the confirmations. That's
fine for testing and an initial deploy, but you'll probably want to switch to a
permanent form of storage for later. Further on in this document you'll see
how.
Next, you need to use it in your @START@ state, and then in a @CONFIRMING@
state. Here's a simple example:
Here's how the above code works:
# First we import the CONFIRM variable so we can use it.
# In our @START@ handler (which is accepts start@(host)) we use the API to send out the confirmation message they should reply-to. Notice how we give a "start" target as the second argument, this is important.
# Then we transition to CONFIRMING and wait for them to reply to that message.
# The user then replies to the message we sent, so we handle the CONFIRMING state. Notice that we are handling "start-confirm-(id_number)" as the initial message, with "start" being the target (2nd parameter) from our above @CONFIRM.send@ call.
# In CONFIRMING we use the @CONFRIM.verify@ method to validate that it's from the right person, to the right target ("start") and that they got the secret (id_number) right.
# Finally, if it's right we send them a welcome message, and if it's not we just ignore the message.
An alternative to ignoring the failed confirmation from them is to cancel it
and go back to the START state for them. The danger with that method though is
a spam bot will get into a loop where you are sending them constant
confirmation messages in a loop between START and CONFIRMING. It's best to
drop it, and maybe provide a "cancel" mechanism.
h2. Using Shelf Storage
Other than a few other methods, there's only a need to change the storage. The
simplest change is to provide a dict-like interface to the
"lamson.confirm.ConfirmationEngine":http://lamsonproject.org/docs/api/lamson.confirm.ConfirmationEngine-class.html
to store. Easiest available is the Python
"shelf":http://docs.python.org/library/shelve.html module which gives a dict
interface to various key/value storage backends.
To use one, just change your code in @config/settings.py@ to be like this:
All you do is create a
"lamson.confirm.ConfirmationStorage":http://lamsonproject.org/docs/api/lamson.confirm.ConfirmationStorage-class.html
and give the @db=@ parameter a dict it can use. Everything else will be the
same.
bq. There might be thread issues with this, and it will definitely fail if you use mulitple processes. See the next
section on using a Django Model.
h2. Using A Django ORM Model
In the "librelist.com":http://librelist.com/ example code you'll find that it
stores the confirmations in the Django model in the @webapp/librelist@
directory. This is actually easily setup, so first read "Hooking Into
Django":/docs/hooking_into_django.html to learn how to access a Django ORM.
After that, you write a simple version of @ConfirmationStorage@ that would look
something like this:
This is from Librelist, so you see we just import the @Confirmation@ model and
then wrap it with the @get@, @delete@, @set@, and @clear@ methods that
@ConfirmationEngine@ needs to run.
For completeness, here's what the Django @Confirmation@ model looks like:
Final step is to configure it in your @config/settings.py@ thusly:
from lamson import confirm
...
from app.model.confirmation import DjangoConfirmStorage
CONFIRM = confirm.ConfirmationEngine('run/pending', DjangoConfirmStorage())
That's all there is to it. This is actually a nice setup because you can use
the Django Admin to manage it during your first deployments.
h2. Other ORM
For other ORM systems simply use the same pattern as the Django example above.
You just create a similar model, wrap it with your own version of
@ConfirmationStorage@ and plug it into the @ConfirmationEngine@ you use.
h2. Targets
The only other thing to understand is why the API has a "target" parameter.
Let's look at the call to CONFIRM.send again:
The "start" string as the second parameter acts as a the target. It says that
this user needs to confirm for the "start" target when they do their reply.
That's why the @route@ on @CONFIRMING@ is then like this:
This way, the user could have multiple simultaneous confirmations going for
different lists and they won't step on eachother.
Without this differentiator, you'd have to either restrict users to just one
confirmation at a time, or you'd end up getting the data all confused.
h2. Confirming Off A Web Link
If you want people to go to a web link instead of simply replying, then you have to do the
following:
# Either write your own version, or subclass @ConfirmationEngine@ so that it uses an address they can't reply to.
# Make sure you use an ORM that can access your database and store both the confirmation info, and each users's state.
# When the user hits the link you give them and does whatever you need, use the web framework's ORM to validate their confirmation.
# Once your web framework has validated their confirmation, then change their state *in Lamson's state* using *your web framework ORM* out of CONFIRMING and into the next state.
Assuming you're doing this all with Python it should be fairly trivial.
h2. Confirm Only By Web Is Bad
I would advise against this method though, since it doesn't really confirm that
the email address you received worked. One of the purposes of doing a
confirmation email exchange is to make sure that this person can both *send*
and *receive*. If you have to point them at your web site, consider having
this process instead:
# Their first interaction with your service sends out an email that sends them to a web page, and transitions them to a @CONFIRMING@ state *but do not send them a confirmation reply address from Lamson.* You'll actually "delay" this until they fill out your web forms.
# In your web framework, you have them fill out forms and such, and then send them the *real* confirmation message using Lamson. Since the Confirmation API is Python you could do this directly in any Python web framework. You're basically moving the call to @CONFRIM.send@ from your @START@ handler into the web framework.
# Then your web framework will have sent them a real confirmation email, not just a link, so when they reply, continue with the usual Lamson confirmation process described above.
Doing it this way ends up being a good balance between too many clicks and replies, but too few to confirm that the end user can actually reply
to email you send them.
h2. The Pending Queue
You should also notice in the above examples that the original message is
stored in a "pending queue" and then given back to you later. This is handy
for either finishing their original request without further intervention, or
inspecting what they original wanted to do. In the original Librelist code I
would take their first message, confirm them, and then pull it out of the
pending queue to send it on. This turned out to not work because socially
people "subscribe" with a garbage first message, but technically it worked
great.
bq. You may want to periodically go through this queue and purge any messages that aren't found in the ConfirmationsStorage. Probably with a simple Python script and a cronjob.
h2. Conclusion
The Lamson confirmation API encapsulates a pattern for confirming potential
users. Feel free to suggest improvements to the API if you find further
patterns that are needed.
lamson-1.0pre11/doc/lamsonproject.org/input/docs/deferred_processing_to_queues.txt 0000644 0000765 0000024 00000017601 11242707504 030223 0 ustar zedshaw staff Title: Deferred Processing To Queues
As of the 0.9.2 release there is preliminary support for deferring handling of
a mail message to a queue for another process to deal with in a separate
handler. This support is rough at this time, but still useful and not too
difficult to configure. As the feature gets more use it will improve and
hopefully turn into a generic "defer to queue" system in Lamson.
What is meant by "defer to queue" is simply that you take messages your state
function receives and you dump them into a maildir queue. You then have
another separate process read from this queue and do the real work.
Potentially you could have many processes deal with this work, and they could
even be on multiple computers.
h2. A More Concrete Example
Imagine that you have a blog posting system and you want to update a big "front
page index" that shows recent posts by your users. However, you don't want to
generate this index on *every* blog post users make, since that could involve
expensive computation and hold up other threads that need to deal with more
urgent email.
The solution is to do the minimum quick processing you can in your POSTING
state function, and then use the
"lamson.queue.Queue":http://lamsonproject.org/docs/api/lamson.queue.Queue-class.html
to queue up messages meant for "front page indexing". Here's how that code
might go:
@route("(post_name)@osb\\.(host)")
def POSTING(message, post_name=None, host=None):
# do the regular posting to blog thing
name, address = parseaddr(message['from'])
post.post(post_name, address, message)
msg = view.respond('page_ready.msg', locals())
relay.deliver(msg)
# drop the message off into the 'posts' queue for later
index_q = queue.Queue("run/posts")
index_q.push(message)
return POSTING
You can see that you just drop it into the queue with @push(message)@ and it's
done. What you don't see is how this then gets picked up by another process to
actually do somehing with this email.
h2. Configuring A config/queue.py
In Lamson you are given control over how your software boots, which gives you
the ability to configure extra services how you need. By default the @lamson
gen@ command just outputs a basic @config/boot.py@ and @config/testing.py@ file
so you can get working, and these will work for most development purposes.
In this tutorial you get to write a new boot configuration and tell Lamson how
to use it. We'll be copying the original boot file over first:
$ cp config/boot.py config/queue.py
Next you want to edit this file so that instead of running an
"SMTPReceiver":http://lamsonproject.org/docs/api/lamson.server.SMTPReceiver-class.html
it will use a
"QueueReceiver":http://lamsonproject.org/docs/api/lamson.server.QueueReceiver-class.html
configured to pull out of the @run/posts@ queue you are using in your @POSTING@
handler.
...
# where to listen for incoming messages
settings.receiver = QueueReceiver(settings.queue_config['queue'],
settings.queue_config['sleep'])
settings.database = configure_database(settings.database_config, also_create=False)
Router.defaults(**settings.router_defaults)
# NOTE: this is using a different handlers variable in settings
Router.load(settings.queue_handlers)
Router.RELOAD=True
...
I've removed the code above the ... and below it since it's the same in the two
files. Notice that you have a @QueueReceiver@ now, and that you are telling
the
"Router":http://lamsonproject.org/docs/api/lamson.routing.RoutingBase-class.html
that it will use @settings.queue_handlers@ for the list of handlers to load and
run.
You now add these two lines to your @config/settings.py@:
...
# this is for when you run the config.queue boot
queue_config = {'queue': 'run/posts', 'sleep': 10}
queue_handlers = ['app.handlers.index']
The @queue_config@ variable is read by the @config/queue.py@ file for the
@QueueReceiver@ and the @queue_handlers@ is fed to the @Router@ as described
above.
h2. Writing The Index Handler
You now have to write a new handler that is in @app/handlers/index.py@ so that
this @config.queue@ boot setup will load it and run it whenever a message hits
the @run/queue@. Here's the code:
from lamson import queue
from lamson.routing import route, stateless
import logging
@route("(post_name)@osb\\.(host)")
@stateless
def START(message, post_name=None, host=None):
logging.debug("Got message from %s", message['from'])
This simple demonstration will just log what messages it receives so you can
make sure it is working.
There are two points to notice about this handler. First, it is marked
@stateless@ because it will run independent of the regular Lamson server, and
you don't want its parallel operations to interfere with your normal server's
state operations. Second, it uses a @Router.defaults@ named @post_name@ that
you would add to your @config.settings.router_defaults@.
Once you have all this slightly complicated setup done you are ready to test
it.
bq. Also note that the examples in the "source releases":/releases/ have code
that does a deferred queue similar to this. Go look there for more code to
steal.
h2. Running Your Queue Receiver
Run your logger and lamson server like normal:
$ lamson log
$ lamson start
Next, go look in your logs and make sure it works by running your unit
tests:
$ nosetests
................
----------------------------------------------------------------------
Ran 16 tests in 1.346s
OK
Your logs should look normal, but now you should see some files in the
@run/posts/new@ directory:
$ ls run/posts/new/
1244080328.M408474P3147Q4.mycomputer.local
That's the results of your @POSTING@ handler putting the messages it receives
into your @run/posts@ maildir queue.
Finally, you'll want to run your queue receiver:
If you're running the code given above then you should see this in the
@logs/lamson.log@ file:
...
DEBUG:root:Sleeping for 10 seconds...
DEBUG:root:Pulled message with key:
'1244080328.M408474P3147Q4.zed-shaws-macbook.local' off
DEBUG:root:Message received from Peer: 'run/posts', From:
'sender-1244080328.22@sender.com', to To
['test.blog.1244080328@osb.test.com'].
DEBUG:root:Got message from sender-1244080328.22@sender.com
DEBUG:root:Message to test.blog.1244080328@osb.test.com was handled by
app.handlers.index.START
Which means your queue receiver is running. You could *in theory* run as many
of these as you wanted, as long as their handlers are stateless.
When you're done you can stop the whole setup with the following command:
$ lamson stop -ALL run
Stopping processes with the following PID files:
['run/log.pid', 'run/queue.pid', 'run/smtp.pid']
Attempting to stop lamson at pid 3092
Attempting to stop lamson at pid 3157
Attempting to stop lamson at pid 3096
h2. Further Advanced Usage
This configuration is debatable whether it is very usable or not, but it works
and will improve as the project continues. To give you some ideas of what you
can do with it:
# Defer activity to other machines or processes.
# Receive messages from other mail systems that know maildir.
# Deliver messages to other maildir aware systems.
# Process messages from a web application, and possibly even generic work.
It might also be possible to actually make your state functions transition to
the queue handler states by simply having the function return the
@module.FUNCTION@ that should be next. Take care with this though as it means
your end user's actions are effectively blocked for that event until the next
run of the queue receiver.
h2. Call For Suggestions
Feel free to offer suggestions in improving this setup (or even better code).
lamson-1.0pre11/doc/lamsonproject.org/input/docs/deploying_lamson.txt 0000644 0000765 0000024 00000016746 11242710567 025475 0 ustar zedshaw staff Title: Deploying Lamson Level 1
These instructions will teach you how to setup a completely clean Python 2.6
installation, a virtualenv with lamson, and all the gear needed to run the
"oneshotblog.com":http://oneshotblog.com/ software on your machine.
You can then "read how to install oneshotblog":/docs/deploying_oneshotblog.html
for yourself.
Most of these instructions could be easily turned into an automated script,
which may happen in the future. For now it is meant to teach you about the
typically dirty details involved in setting up a system for the first time. It
also tries to avoid various problems with different operating systems, so let
me know how it works for you.
h2. A Warning
Deploying server software is a notoriously nasty process, especially the first
10 or 20 times. Most operating systems do their best to enforce completely
arbitrary restrictions on your file layouts and configurations, and every
system has different arbitrary restrictions.
When you go through these instructions, make sure you stay awake and be ready
to delve into why a particular step might not work on your system. There's a
good chance you missed something or that there's something just slightly
different about your system that makes the step not work.
For example, in the parts of this document where I setup
"oneshotblog.com":http://oneshotblog.com/ I ran into a problem with
"SpamBayes":http://spambayes.sourceforge.net/ dying because it couldn't iterate
a bsddb. Problem is this works just fine in the exact same setup on a CentOS
machine and was only dying on a MacOSX machine that I later tested. For
whatever reason, the exact same setup can't run SpamBayes on OSX, even though
it can run with the stock Python 2.5 in OSX.
To solve the problem I just had to show you how to disable SpamBayes in the
oneshotblog.com code so you could test it. That's just how deployment goes.
You get on a machine and start setting things up and then 2/3 of the way
through the configuration you find out that something doesn't work.
Only choices are to work around the problem (like I did) or try to figure out
why your machine is different and fix it.
h2. Step 0: Setup A Workplace
You'll want a directory to do this in so that you don't screw up your machine.
Here's what I did:
$ mkdir deploy
$ cd deploy
$ export DEPLOY=$PWD
That last bit is so you can refer to this deployment directory with $DEPLOY
(which I'll be using in the instructions from now on).
h2. Step 1: Get Python
Many operating systems have old versions of Python, and even though Lamson
works with 2.6 or 2.5, you'll probably want to get 2.6 for your deployment. If
your OS has 2.6 available then go ahead and install it.
If it doesn't have the right Python version, then here's how you can install it
from source and use it as your default Python. To do this, just punch in these
commands:
$ mkdir source
$ cd source
$ wget http://www.python.org/ftp/python/2.6.2/Python-2.6.2.tgz
$ tar -xvf Python-2.6.2.tgz
$ cd Python-2.6.2
$ ./configure --prefix=$DEPLOY
$ make
$ make install
After this you will have a bunch of new directories in $DEPLOY:
$ ls $DEPLOY
bin include lib share source
Finally, you just have to put this new bin directory into your $PATH:
$ export PATH=$DEPLOY/bin:$PATH
... then you just try it out to make sure that you have the right one:
Python-2.6.2 $ which python
$DEPLOY/deploy/bin/python
Python-2.6.2 $ python
Python 2.6.2 (r262:71600, Jun 8 2009, 00:44:56)
[GCC 4.0.1 (Apple Inc. build 5490)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> ^D
That's it, you'll now be able to use this Python when you need to run your
Lamson server, and setup a virtualenv (coming next) so that you're walled off
from the rest of the system.
bq. Operating system fanatics will scoff at putting the python install in this
directory, so if you want you can just install it to the default /usr/local on
your system and deal with all the various clashes and conflicts you'll have,
especially if you are on an MacOSX machine.
h2. Step 2: Install VirtualEnv
Now we need to create a "virtual environment" to install all your software. To
do this we'll need easy_install installed to your $DEPLOY directory:
$ cd $DEPLOY/source
$ wget http://peak.telecommunity.com/dist/ez_setup.py
$ python ez_setup.py
$ which easy_install
$DEPLOY/bin/easy_install
As you can see, you now have a clean install of easy_install in your fresh
$DEPLOY/bin directory for you to use. Now you need to install @virtualenv@:
$ easy_install --prefix $DEPLOY virtualenv
$ which virtualenv
$DEPLOY/bin/virtualenv
bq. Make sure you use @--prefix $DEPLOY@ above or you'll install things into
the default system setup even though easy_install is clearly and obviously
running from a Python in a totally different location so easy_install should
know that.
h2. Step 3: Create Your VirtualEnv
With that you are ready to setup your virtual environment which will house your
Lamson setup and fill it with the gear you need.
First up is getting your virtualenv created and activated:
$ cd $DEPLOY
$ virtualenv LAMSON
New python executable in LAMSON/bin/python
Installing setuptools............done.
$ cd LAMSON
$ . bin/activate
That's pretty simple, and it tells you clearly that you are using the LAMSON
virtualenv. It prepends that to your currently prompt, so your prompt may look
different.
After that we can use easy_install to install our packages to this LAMSON
virtual env. Keep in mind that these packages will be in $DEPLOY/LAMSON, so
they won't infect your regular $DEPLOY setup.
$ cd $DEPLOY/LAMSON
$ easy_install lamson
After that, you have lamson installed and ready to go, and you can install
anything you want, but there is one catch:
bq. You *MUST* be in the $DEPLOY/LAMSON directory or easy_install barfs
complaining that the package is not there.
h2. Step 4: Making Sure It Works
All of this setup is pointless if you can't get back to it later, so exit your
terminal completely and start a new one so you can do this:
$ cd projects/lamson/deploy/
$ export DEPLOY=$PWD
$ export PATH=$DEPLOY/bin:$PATH
$ cd $DEPLOY/LAMSON
$ . bin/activate
(LAMSON) $ which python
$DEPLOY/deploy/LAMSON/bin/python
(LAMSON) $ which easy_install
$DEPLOY/deploy/LAMSON/bin/easy_install
(LAMSON) $ which lamson
$DEPLOY/deploy/LAMSON/bin/lamson
(LAMSON) $ cd $DEPLOY
(LAMSON) $ lamson help
If you can do all that, then you know you've got the setup going, now you just
need a little shell script to kick this all into gear automatically:
#!/bin/sh
export DEPLOY=$PWD
export PATH=$DEPLOY/bin:$PATH
cd $1
source bin/activate
cd $DEPLOY
To use this script, you just do this:
$ cd projects/lamson/deploy
$ . activate LAMSON
With that you have a fully ready to go setup that's not using your normal
system's Python at all, has Python 2.6 installed, a fully virtualenv, and the
start of your lamson setup.
h2. Conclusion
Your next step is to try and setup "oneshotblog":http://oneshotblog.com/ using
the "instructions I've written":/docs/deploying_oneshotblog.html to follow
these instructions.
This document is very fresh, so send me feedback on your experience with
running through it. Make sure you tell me what system you are on and that you
ran each command exactly when you do.
lamson-1.0pre11/doc/lamsonproject.org/input/docs/deploying_lamson_level_2.txt 0000644 0000765 0000024 00000000060 11243027730 027055 0 ustar zedshaw staff Title: Deploying Lamson Level 2
Coming soon...
lamson-1.0pre11/doc/lamsonproject.org/input/docs/deploying_librelist.txt 0000644 0000765 0000024 00000000054 11243027650 026151 0 ustar zedshaw staff Title: Deploying Librelist
Coming soon...
lamson-1.0pre11/doc/lamsonproject.org/input/docs/deploying_oneshotblog.txt 0000644 0000765 0000024 00000022614 11242710431 026504 0 ustar zedshaw staff Title: Deploying OneShotBlog
These instructions follow from "Deploying Lamson Level 1":/docs/deploying_lamson.html and you should follow those first before attempting these. If you run into
problems with these instructions, then "email the lamson@librelist.com":mailto:lamson@librelist.com mailing list for help.
h2. Step 5: Setting Up The OneShotBlog
Let's see if we can setup the OneShotBlog example from the Lamson source the
way it is on the "oneshotblog.com":http://oneshotblog.com site. We'll need a
few more modules installed with easy_install:
Let's grab the 0.9.3 source from "PyPI":http://pypi.python.org/pypi/lamson/0.9.3 so we can get at the OSB example source:
$ cd $DEPLOY/source
$ wget http://pypi.python.org/packages/source/l/lamson/lamson-0.9.3.tar.gz
$ tar -xzf lamson-0.9.3.tar.gz
$ cd lamson-0.9.3/examples/osb
Now we hit a slight snag. OSB is using
"SpamBayes":http://spambayes.sourceforge.net/ to do spam filtering, but you
probably will have a broken setup and would need to configure a ton of stuff to
get it working. For now we're, just going to cheat, since it looks like
SpamBayes has problems with trying to iterate the keys in a bsddb under *some*
Python builds. To avoid the problem, we're just going to edit
@app/handlers/comment.py@ to remove the line with @spam_filter@:
@route("(user_id)-AT-(domain)-(post_name)-comment@(host)")
# DELETE THIS LINE IN app/handlers/comments.py
@spam_filter(SPAM['db'], SPAM['rc'], SPAM['queue'], next_state=SPAMMING)
def START(message, user_id=None, post_name=None, host=None, domain=None):
comment.attach_headers(message, user_id, post_name, domain)
confirmation.send(relay, "comment", message, "mail/comment_confirm.msg", locals())
return CONFIRMING
The spam filtering does work, but SpamBayes is difficult to get working in such
a small test run.
You are now sitting in the OSB example code, so you can fire up the logger
server and run the unit tests to make sure everything is working:
..............
======================================================================
FAIL: handlers.comments_tests.test_spam_sent_by_unconfirmed_user
----------------------------------------------------------------------
Traceback (most recent call last):
...
-------------------- >> begin captured logging << --------------------
root: WARNING: Attempt to post to user 'spamtester@somehost.com' but user doesn't exist.
--------------------- >> end captured logging << ---------------------
======================================================================
FAIL: handlers.comments_tests.test_spam_sent_by_confirmed_user
----------------------------------------------------------------------
Traceback (most recent call last):
...
-------------------- >> begin captured stdout << ---------------------
run/posts count after dever 1
run/posts count after dever 2
--------------------- >> end captured stdout << ----------------------
-------------------- >> begin captured logging << --------------------
root: WARNING: Attempt to post to user 'spamtester@somehost.com' but user doesn't exist.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 25 tests in 1.363s
Those are just fine since you don't have PyEnchant installed and aren't using
the spam filtering.
h2. Step 6: Run OneShotBlog Example
Now you're running the logger server and have your unit tests going, and
hopefully you can fix anything that you run into by now. All you need now is
to run the whole setup and try it out:
With all this gear running you should be able to look in the @logs/lamson.log@
and @logs/logger.log@ to see what's going on. You'll see the following
activity:
* Forwarding receiver taking mail that couldn't be delivered and forwarding it to the logger server.
* The queue receiver pulling messages off run/posts and either delivering them as comments or updating the index.
* The rest of lamson processing mail and doing its job of feeding these two or just sending emails.
bq. If you want to see the configuration for these two other servers look in @config/queue.py@ and
@config/forward.py@ or better yet, diff them against @config/boot.py@ to see what's really different.
You should also check to see that they are really running:
h2. Step 7: Playing With OneShotBlog
Now we get to play with it. Lamson comes with a web server that you can run to
do simple testing so start up a second window/terminal and do this:
$ cd projects/lamson/deploy/
$ . activate LAMSON
(LAMSON) $ cd source/lamson-0.9.3/examples/osb/
(LAMSON) $ lamson web -basedir app/data
Starting server on 127.0.0.1:8888 out of directory 'app/data'
Now hit "http://localhost:8888/":http://localhost:8888/ with your browser and
see the junk left over from your test runs. Most of those posts won't actually
exist, so let's make a fake one for now.
You can forget about this web server window for now, and go back to your LAMSON
window to do this with mutt:
# mutt -F muttrc to get it going with a fake setup.
# Send an email to first.blog@oneshotblog.com (m is the key).
# You'll get a confirmation back, reply to it.
# You'll get a welcome message, but this message isn't in the index yet. You can go look at it directly though.
# Send *another* email, this time to my.new.post@oneshotblog.com.
# No confirmation this time, just a message saying it was completed.
# *Now* go look at the index (might take a few seconds, up to 10).
# Click on the post title and go look at it.
# Right click on the [send comment] link and copy the email address.
# Go back to mutt and send an email to that address, this will post a comment to that post.
# Reply to the comment confirmation email, this should be the only one you get.
# In about 10 seconds you'll see your comment show up.
With that you have fully tested out the OneShotBlog example. All that remains
would be a full deployment in a for-real situation, which is what we'll do
next.
h2. Step 8: Running On Port 25 For Real
The only problem with testing out the OneShotBlog with your own email client is
that you need to trick your computer into thinking your localhost address is
also oneshotblog.com. To do that, open your /etc/hosts file and make whatever
changes you need to have localhost be oneshotblog.com also.
You'll know you've got it right when you can point your browser at
"oneshotblog.com":http://oneshotblog.com/ and see your @lamson web@ window
display log messages showing you click around.
bq. Remember to undo this or you might be annoyed later.
Next, you'll need to stop lamson and restart it to use port 25. This will be a
problem if you have another server running on port 25, so make sure you turn
that server off for now.
bq. I hope you aren't doing this on a live site where that's a problem.
$ lamson stop
$ vim config/settings.py
At this point you'll want to change the receiver_config to look like this in
@config/settings.py@:
Then you'll want to restart lamson so that it drops privilege to your user
after grabbing that port. Easiest way to find out what your user id (uid) and
group id (gid) are is to use Python:
Python 2.6.2 (r262:71600, Jun 8 2009, 00:44:56)
[GCC 4.0.1 (Apple Inc. build 5490)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import os
>>> os.getuid()
500
>>> os.getgid()
500
>>> ^D
This shows that I'm uid 500 and gid 500, so now I can start my server:
That last command just makes sure that lamson didn't accidentally change the
permission of a stray file or queue to @root@.
With that you should be running your lamson server as your user rather than as
root, but still bound to port 25. Here's how to check:
$ ps aux | grep lamson | grep root
If you don't see anything printed out, then you're safe. If you see something
running as root, then you've got some work to do.
h2. Conclusion
I'll leave it to you to actually get your mail client to talk to this
OneShotBlog and working. If you have problems, look at all the files in
@logs/@ and also use the @lamson queue@ command to inspect the different queues
in the @run/@ directory.
At this point, you should know enough about setting up a lamson server and
configuring a real application (warts and all). You should also have learned
how to get a clean Python installation that you can use no matter what your
native OS does to Python, even if it's retarded and renames python to Python
for no apparent reason.
lamson-1.0pre11/doc/lamsonproject.org/input/docs/faq.txt 0000644 0000765 0000024 00000033652 11242712465 022673 0 ustar zedshaw staff Title: Frequently Asked Questions
If a question is missing you can "email the lamson@librelist.com mailing
list":mailto:lamson@librelist.com about it and I'll answer.
h2. What is Lamson?
Lamson is a pure Python SMTP server designed to create robust and complex mail
applications in the style of modern web frameworks such as Django. Unlike
traditional SMTP servers like Postfix or Sendmail, Lamson has all the features
of a web application stack (ORM, templates, routing, handlers, state machines,
Python) without needing to configure alias files, run newaliases, or juggle
tons of tiny fragile processes. Lamson also plays well with other web
frameworks and Python libraries.
h2. Where does the name "Lamson" come from?
"Lamson Tubes" is a colloquial name for Pneumatic Tubes which were used last
century to deliver mail, packages, and hazardous material to the corporate
world. They are still in use today.
h2. What kind of applications do you envision being built on top of Lamson?
Why, spam of course (like I could stop that). As well as spam fighters,
greylisters, "campaign management" applications, mail firewalls, mime strippers
(for those Exchange file shares), help desk support applications, games,
mailing lists ('cause everyone loves writing those), and even SMTP portions of
just about any web site.
A few examples of applications that are actually written using Lamson (with
source included in the "source releases":/releases/
* "OneShotBlog":http://oneshotblog.com/
* "Librelist":http://librelist.com/
* "MyInboxIsNotA.TV":http://myinboxisnota.tv/
With many more to come.
h2. How do I install lamson on a Debian or CentOS server?
Debian and CentOS are notorious for being dinosaurs. Both distributions of
Linux suffer from the false rationale that older software is more stable and
secure. The reality is that the stability or security of a piece of software
is not a function of its age, and in many cases the newer versions of software
will typically fix many stability and performance problems. Despite this fact,
these two variants of Linux are notorious for back-porting patches from later
versions to older versions rather than just using the newer version.
In order to help people who need to run a modern piece of software on their
antiquated operating systems, I've written an extensive document on "how to
Deploy Lamson":/docs/deploying_lamson.html that tells you how to do it in a way
that should work reliably on most Unix platforms, even if they have an older
version of Python.
h2. Is there a mailing list?
Yes, and it's written in Lamson. You can send an email to "lamson@librelist.com":mailto:lamson@librelist.com
and you'll be able to join. The code for this mailing list system is also included in the "source releases":/releases/
so you can learn from it.
h2. Is there an IRC channel?
Yes, and it has relatively low traffic. You can join #librelist on irc.freenode.org.
h2. How do I report a BUG/Feature/Question in Lamson?
Currently the best place to report a Lamson bug is to the "lamson@librelist.com":mailto:lamson@librelist.com
mailing list. If you don't want to subscribe to the list to just report a bug, then you can
"contact me":/contact.html directly to report it.
h2. How do I work on Lamson?
I currently do all the work on Lamson myself. I don't want to discourage contributions, but I've
found that when a project is small and just getting started it's best to keep it under the control
of one person.
If you find bugs, then please report them to the "lamson@librelist.com":mailto:lamson@librelist.com mailing
list to "to me directly":/contact.html and I'll fix them up.
h2. How do I try out (install) Lamson?
Best way to do it is to have
"easy_install":http://peak.telecommunity.com/DevCenter/EasyInstall and simply
do:
$ sudo easy_install lamson
Then you'll get it installed and can play with it. Refer to the "getting
started":/docs/getting_started.html documentation for more information.
h2. How can I get the code to Lamson?
Lamson lives on Launchpad at "https://launchpad.net/lamson":https://launchpad.net/lamson where you can get the code via:
bzr branch lp:lamson
Bazaar may ask you to login, but it should still give you the source.
Refer to the "download instructions":/download.html for more information.
h2. What features are planned for the 1.0 release?
I try to make "1.0" what most other people would consider 80% complete. This
lets people get it and work with it, and then I can refine it for the 2.0
release with 100% of features people actually use.
With this in mind, I'm planning on the 1.0 release to be based on the sample
applications and my own applications and include the smallest set of features.
The most important part of the 1.0 release will be good documentation and bug
free high quality code.
h2. What are Lamson's current features?
Lamson is currently running a handful of sites and is almost ready for a 1.0 release.
It has these high level features:
* Very good bounce detection and analysis.
* Spam blocking.
* Get an application going quickly with the "lamson gen" command.
* Full set of "self-documented commands":/docs/lamson_commands.html for managing and developing a lamson application.
* Lots of "documentation":/docs/ with more to come.
* Sample applications for you to base your work on.
* Handle mail for arbitrary hosts and addresses using simple regex routing.
* Process requests including full access to Python's complete MIME email libraries.
* Absolutely awesome full conversion to Unicode including complete cleaning of incoming mail.
* Routing requests in a standard web application framework style based on addresses.
* Processing mail messages (requests) in either simple generic handlers, or in more complex and robust state machines.
* Use any Python database storage you want.
* Use of Jinja2 or Mako templates, with Jinja2 as the default.
* Craft and send plain text or HTML email including attachments, with a great HTML mail generation API that even "knits" in CleverCSS.
* Unit test helpers for having a conversation, checking spelling, etc.
* Defer processing to Maildir queues for offline handling of bigger tasks.
* Extensive logging and development tools for debugging your email applications.
* Mutt configurations to fake out mutt so that it talks to your server.
* 100% code coverage in the unit tests.
h2. Isn't Postfix/Sendmail/Exim Faster?
That is a tough question to answer actually. If all you need to do is receive
and deliver email then a well established traditional email server like Postfix
or Exim (please don't use Sendmail) is the way to go. Hands down these servers
are the fastest and the best at this job.
However, if you need to actually do something smart with your email, like
manage many mailing lists or handle support requests, then these servers are
definitely slower.
The reason is they require that you configure them to take messages they've
already received and hand them to a separate process like a Perl, Python, or
Ruby script. This separate process then has to parse the message *again*, do
its job without stepping on any other processes that might be running (that
means locks), and then send response messages back to the server for even more
SMTP parsing.
With the triple and sometimes quadruple MIME parsing, the heavy weight
processes, the difficult to manage locking, and the additional configuration
headaches, there's no way traditional mail servers beat Lamson in speed.
Lamson only processes a message once, maybe twice if you defer to a queue.
Once the message is parsed you get full access to Python immediately, without
spawning a separate process. Even if you defer to a queue, the Lamson dequeue
server stays resident and processes the queue without forking. You can even
run many dequeue servers on mulitple machines processing a shared Maildir if
you need the extra processing.
In the end, threads and function calls beats processes and pipes.
h2. Why not use Sendmail's Milter?
Sendmail has a protocol named "Milter" that lets you write a mail processing
server that acts as a sort of "slave" to the sendmail process. This protocol
is supported by at least Postfix as well, maybe other servers.
Feel free to go try Milter. When you're done trying to figure out the protocol
from the dense C code, configure the m4 macros, find a decent milter protocol
library that doesn't involve installing sendmail, and debugging the final
setup, then you can come back and have it easy with Lamson.
h2. Why does Lamson send messages to a relay host?
Lamson doesn't have to deliver to a relay host, but it is a smarter more
practical use of the technology.
Lamson is written in Python and does actually run slower than the established
mail servers. In addition, Lamson is hopefully doing something more than just
routing email around to people. It is probably processing messages, crafting
replies, querying databases, hitting REST interfaces, and all the other things
you'd want to do with a modern application. This takes time and resources and
are probably more valuable operations than just simple delivery.
For this reason, you want to use a dumb workhorse like Postfix to do your
actual delivery, and reserve the smart processing that has value for Lamson.
h2. What about security?! Shouldn't Lamson be 20 processes?
Have you ever asked why other mail servers are a bundle of a billion processes?
Why have one server receiving mail, another routing it, and another handing it
to users?
The answer is back in the 1970's most mail was delivered to Unix users in their
home directories or similar files that required special access rights to
modify. Also in the 1970's special ports like 25 for SMTP required root
access, which in the tiny Internet of the time meant that the server could be
"trusted". These two realities of the time meant that to receive and deliver
mail at least some part of the system had to run as root. To keep things safe,
modern mail systems reduce the amount of time spent as the root user by
separating their functionality into different processes.
However, if you never have to deliver to a user, and all you ever do is process
mail and talk to other servers like RDBMS, then why do you need all this
privilege separation? Sites run just fine with systems running as one or two
processes without the complexity of some illogical privilege separation getting
in their way.
To put this into perspective, imagine that you were writing a Django
application and you were required to have a separate process for the HTTP
layers, the view layer, the model layer, the HTTP responses, and the RDBMS
access layers? Each one required a different user, a different configuration
file, and you needed another process just to keep them all sane. All of this
just so that if someone hacks into your HTTP server as root they supposedly
can't cause any damage.
Yet, they are on your server as root after all.
In practice, you can run Lamson as a separate root process, and then use
another "dequeue server" to do the real processing, if you feel you need that
security.
But, consider delaying that decision until you absolutely need it, because the
security benefits aren't worth the development and deployment hassles.
h2. How come nobody thought of this before?
I don't know why, since it did seem kind of simple. There's at least one other
project written in Perl called "qpsmtpd":http://smtpd.develooper.com/ that does
something similar, and there may be more. If you know others feel free to
"contact me":/contact.html and let me know.
h2. Isn't [Insert Random Java Mail Server] actually the first mail "framework"?
I get this quite frequently when I make the claim that Lamson is the first
email framework, and it may be true that there was a framework out there before
Lamson. The internet is a big place, so anything is possible. However, I
looked really hard and I couldn't find a single *modern* mail framework. All
that existed were servers I could use to build a framework.
You see, around 2003 or 2004 the concept of "framework" changed. Before then
all you needed was a server with an extension API named so that it rhymed with
"Servlet". As long as your server provided a way to drop a class into the
processing queue and let a programmer handle the request you could call that a
framework.
The usual end result for these *servers* is that you could use them to build a
framework if you wanted, but what you'd get is affectionately called a
"frankenstack". You'd grab an ORM from here, a template system from there,
maybe a workflow engine, write a Maven or Ant script to manage it, and wire it
all together with some lame secret sauce code you think gives you a competitive
edge.
Then along came the modern frameworks like Django and Rails that included
everything you needed in a bundle that you could use right away. They had ORM,
templating, routing, higher level request processing, email support, REST
support, and anything else you might need for the 80% of your application you
don't care about.
Some people prefer less of these defaults, some people more, but nearly
everyone who has to get a project done prefers more than just an extension API
so they can build their own framework.
Today if you try to claim "Apache James":http://james.apache.org/ is a
framework you'd be wrong. I could *build* a framework with it, but I could
just as easily build that same framework with Python, Ruby, sendmail and even
postfix. James and friends are just servers, not frameworks. In fact, my
experience with James and similar Java mail servers is they are much harder to
use than aliases+pipes in Postfix.
I now advocate that if your framework doesn't at least support data, views, and
high level logic as first class entities then it's just a server. You don't
have to use ORM, any particular templating, or Finite State Machines like
Lamson does. You don't even have to settle on only one way to do data, views,
and logic.
You *must* at least support data, views, and logic out of the box so your user
doesn't have to go shopping at "APIs-R-Us" just to use your gear.
lamson-1.0pre11/doc/lamsonproject.org/input/docs/filtering_spam.txt 0000644 0000765 0000024 00000013430 11242713153 025112 0 ustar zedshaw staff Title: Filtering Spam With Lamson
Lamson supports initial use of the
"SpamBayes":http://spambayes.sourceforge.net/ spam filter library for filtering
spam. What Lamson provides is a set of easy to use decorators that you attach
to your state functions which indicate that you want spam filtered. It also
uses the default SpamBayes configuration files and database formats as you
configure, so if you have an existing SpamBayes setup you should be able to use
it right away.
h2. Using lamson.spam
Lamson gives you a simple decorator to place on any state functions that should
block spam. Typically you do *not* want spam filtering on your entire
application, since that would prevent legitimate registrations and put too much
burden on your system. It's better to put spam filtering on the "insider"
parts, and to have confirmation emails on "outsider" pieces.
Instead, what you want is to indicate that your "choke points" are filtering
spam using
"lamson.spam.spam_filter":http://lamsonproject.org/docs/api/lamson.spam.spam_filter-class.html
so that when a spam is received they are put into a "spam black hole".
Here's an trivial example where the user is in the POSTING state, and you want
everything to work like normal, but if they spam then you flip them into a
SPAMMING state.
The line to look at is obviously the @spam_filter@ line, which tells Lamson that you will:
# Use the SpamBayes training database @run/spamdb@ for the detection.
# Use the SpamBayes @run/.hammierc@ file for your config (optional and ignored if it is not there).
# Use @run/spam@ as the dumping ground for anything classified as spam.
# The next_state to transition to if they send a spam message. *This is optional, but very helpful.*
With this, the @spam_filter@ then wraps your state function, and every message
is fed to SpamBayes. If SpamBayes says it's spam then Lamson will dump it into
your @run/spam@ and transition to SPAMMING *without running your POSTING
state*.
Once you are in this new @SPAMMING@ state (or any state you like) you can do
whatever you want. You can leave them there, or you can have an external tool
that let's you un-block someone. Pretty much any spam handling scheme you want
is available.
Since your spam is placed into a queue you can inspect it later and check for
any accidentally miscategorized mail, then use the SpamBayes tools to retrain
for the misdetection.
bq. Lamson only classifies mail that is marked as actual spam by looking at
the 'X-Spambayes-Classification' header and seeing if it starts with 'spam'.
If it is 'unsure' or 'ham' it will let it through.
h2. Effectiveness
"I've":http://zedshaw.com/ been running a variant of this since the middle of
May 2009 and it works great. The code I run is a custom version that fits the
weirdness of my email setup but the principles are the same. I'm currently
using the above spam filtering, some gray listing, and a few other tricks to
block most of my incoming spam.
With all the spam block measures I've managed to cut down my spam to about 2-3
a day out of about 100-200 I receive. The majority of the "spam" that gets
through is actually email that's classified as "unsure" which I then use to
retrain SpamBayes to make it stronger.
However, that's my personal server, so in the case of a Lamson application
you'll want to be careful that your spam blocking activities don't prevent too
much legitimate use.
h2. Changing What "Spam" Means
You can also change how spam is determined by sub-classing
"lamson.spam.spam_filter":http://lamsonproject.org/docs/api/lamson.spam.spam_filter-class.html
and doing your own implementation of the @spam@ method.
h2. Using SpamBayes
An important point about SpamBayes is that it comes with all the command line
tools you need to configure and train your database using a corpus of spam you
might have. All Lamson needs to do is read this database to determine if it is
spam or not.
With mutt, I save the message to "=spam", which places the spam in Mail/spam
along with all of the others. Then I run this command:
sb_mboxtrain.py -s ~/Mail/spam -d run/spamdb
This goes through the spam mailbox, and any emails that SpamBayes has *not*
already classified get used for training.
SpamBayes comes with other commands you can "read
about":http://spambayes.sourceforge.net/docs.html on their site (if you can
find it).
h2. Autotraining
Lamson doesn't support "autotraining" directly, since it's not clear in each
situation what is obviously spam. In my personal setup I know that any email
not for registered users is obviously spam, so I can autotrain those.
If you want to implement autotraining for a part of your application, then look
at the API for
"lamson.spam.Filter":http://lamsonproject.org/docs/api/lamson.spam.Filter-class.html
and simply use it in the right state function.
h2. Configuration
Finally, the above sample code is not the best way to configure the spam filter.
It's better to put the configuration in @config/settings.py@ and simply reference
it from there.
In your @config/settings.py@ put this:
from config.settings import SPAM
@spam_filter(SPAM['db'], SPAM['rc'], SPAM['queue'], next_state=SPAMMING)
def START(message, ...):
# this is the better way to do your config
With that you can then change up the configuration as needed in
your deployments without having to change your code.
lamson-1.0pre11/doc/lamsonproject.org/input/docs/getting_started.txt 0000644 0000765 0000024 00000104617 11242713461 025310 0 ustar zedshaw staff From: Zed
Title: Getting Started With Lamson
Lamson is designed to work like modern web application frameworks like Django, TurboGears,
ASP.NET, Ruby on Rails, and whatever PHP is using these days. At every design decision
Lamson tries to emulate terminology and features found in these frameworks. This Getting Started
document will help you get through that terminology, get you started running your first
lamson application, and walk you through the code you should read.
In total it should take you about 30 minutes to an hour to complete. If you just want
to try Lamson, at least go through the 30 *second* introduction given first.
h2. The 30 Second Introduction
If you have Python and "easy_install":http://peak.telecommunity.com/DevCenter/EasyInstall already, then try this out:
$ easy_install lamson
$ lamson gen -project mymailserver
$ cd mymailserver
$ lamson start
$ lamson log
$ nosetests
$ lamson help -for send
$ lamson send -sender me@mydomain.com -to test@test.com \
-subject "My test." -body "Hi there." -port 8823
$ less logs/lamson.log
$ mutt -F muttrc
You now have a working base Lamson setup ready for you to work on with the following installed:
* Lamson and all dependencies (Jinja2, nosetests)
* Code for your project in mymailserver. Look in app/handlers and config/settings.py.
* Two initial tests that verify your server is not an open relay and forwards mail in tests/handlers/open_relay_tests.py.
* A "logger" server running on port 8825 that dumps all of its mail into the run/queue maildir.
* A config script for mutt (muttrc) that you can use to inspect the run/queue *and* also send mail using Lamson's *send* command.
When you're in mutt during the above test run, try sending an email. The
included muttrc is configured to use the run/queue as the mail queue, and to
use the @lamson sendmail@ command to deliver the mail. This tricks mutt into
interacting directly with your running Lamson server, so you can test the thing
with a real mail client and see how it will work without having to actually
deploy the server.
Finally, if you wanted to stop all of above you would do:
$ lamson stop -ALL run
Which tells Lamson to stop all processes that have a .pid file in the @run@ directory.
h2. Important Terminology
If you are an old SMTP guru and/or you've never written a web application with
a modern web framework, then some of the terminology used in Lamson may seem
confusing. Other terms may just confuse you or scare you because they sound
complicated. I tried my best to make the concepts used in Lamson
understandable and the code that implements them easy to read. In fact, you
could probably read the code to Lamson in an evening and understand how
everything works.
Experience has taught me that nobody reads the code, even if it is
small. Therefore, here are the most important concepts you should know to get
a grasp of Lamson and how it works.
* MVC(Model View Controller) -- Model View Controller is a design methodology used in web application frameworks where the data (model), presentation (view), and logic (controller) layers of the application are strictly separated.
* FSM(Finite State Machine) -- Lamson uses the concept of a Finite State Machine to control how handlers execute. Each time it runs it will perform an action based on what it is send *and* what it was doing last. FSM in computer science class are overly complex, but in Lamson they are as easy to use as a @return@ statement.
* Template -- Lamson generates the bodies of its messages using Templates, which are text files that have parts that get replaced with variables you pass in. Templates are converted to their final form with a process called *rendering*.
* Relay -- The *relay* for a Lamson server is where Lamson delivers its messages. Usually the Relay is a smart tougher server that's not as smart, but very good at delivering mail. Lamson can also be run as a Relay for testing purposes.
* Receiver -- Lamson typically runs as the Receiver of email. If you are familiar with a web application setup, then Lamson is the inverse. Instead of Lamson runing "behind" an Apache or Nginx server, Lamson runs "in front" of an SMTP server like Postfix. It listens on port 25, handles the mail it should, and forwards the rest to the Relay. This makes Lamson much more of a Proxy or filter server.
* Queue -- Lamson can also do all of its processing off a queue. In this setup you would have your normal mail server dump all mail to a maildir queue, and then tell Lamson to process messages out of there. This can be combined with the usual Receiver+Relay configuration for processing messages that might take a long time.
* Maildir -- A standard created for the qmail project with stores mail in a directory such that you can access the mail atomically and store it on a shared disk without conflicts or locking.
h2. Managing Your Server
Your Lamson application is now running inside the Lamson Python server. This is a very simple server based on Python's
"smtpd":http://docs.python.org/library/smtpd.html and "asyncore":http://docs.python.org/library/asyncore.html libraries.
bq. If you want to know more about how it operates, take a look at the @lamson/server.py@ file in the source distribution.
You'll need to use a few Lamson commands to manage the server. You already experienced them in the
30 second introduction, and you can review "them all":/docs/lamson_commands.html or see them
by using the @lamson help@ command.
Right now you have Lamson running on port 8823 and a "Lamson logger" running on 8825. This means that
your lamson server (port 8823) will forward its messages to the logger (port 8825) thinking it's your
real relay server. The truth is the logger just logs its messages to logs/logger.log and dumps it into
run/queue so you can inspect the results.
Before we learn how to manage them and what they do, open up the
@config/settings.py@ file and take a look:
Your file probably has some comments telling you what these do, but it's important to understand
how they work.
First, this file is just plain old Python variables. It is loaded by one of
two other files in your config directory: @config/boot.py@ or
@config/testing.py@. The @config/boot.py@ file is started whenever you use the
@lamson start@ command and its job is to read the @config/settings.py@ and
start all the services you need, then assign them as variables back to
@config.settings@ so your handlers can get at them. The @config/testing.py@ is
almost the same, except it configures @config.settings@ so that your unit tests
can run without any problems. Typically this means setting the spell checker
and *not* starting the real server.
bq. Lamson can load any boot script you like, see "Deferred Processing To
Queues":/docs/deferred_processing_to_queues.html for an example of using this
to make a queue processor.
The important thing to understand about this setup (where a boot file reads settings.py and
then configures @config.settings@) that it makes it easy for you to change Lamson's operations
or start additional services you need and configure them. For the most part you won't need
to touch @boot.py@ or @testing.py@ until you need to add some new service, change the template
library you want to use, setup a different database ORM, etc. Until then just ignore it.
h2. settings.py Variables
The @receiver_config@ variable is used by the _lamson start_ command to figure out where to listen
for incoming SMTP connections. In a real installation this would be port *25* on your external
IP address. It's where the internet talks to your server.
The @relay_config@ setting is used by Lamson to figure out where to forward message replies (responses)
for real delivery. Normally this would be a "smart host" running a more established server
like "Postfix":http://www.postfix.org/ or "Exim":http://www.exim.org/ to do the grunt work
of delivering to the final recipients.
The @handlers@ variable lists the modules (not files) of the handlers you want to load.
Simply put them here and they'll be loaded, even the "lamson.handlers":http://lamsonproject.org/docs/api/lamson.handlers-module.html
modules will work here too.
The @router_defaults@ are for the "lamson.routing.Router":http://lamsonproject.org/docs/api/lamson.routing.RoutingBase-class.html
class and configure the default
routing regular expressions you plan on using. Typically you'll at least configure the
@host@ regular expression since that is used in every route and shouldn't change too often.
Finally, @template_config@ contains the configuration values for the templating system you'll
be using. Lamson supports either Mako or Jinja2, but defaults to Jinja2.
h2. Looking At config/boot.py
Programmers need to know how everything works before they trust it, so let's look at the _config/boot.py_
file and see how these variables are used:
from config import settings
from lamson.routing import Router
from lamson.server import Relay, SMTPReceiver
from lamson.utils import configure_database
from lamson import view
import logging
import logging.config
import jinja2
# configure logging to go to a log file
logging.config.fileConfig("config/logging.conf")
# the relay host to actually send the final message to
settings.relay = Relay(host=settings.relay_config['host'],
port=settings.relay_config['port'], debug=1)
# where to listen for incoming messages
settings.receiver = SMTPReceiver(settings.receiver_config['host'],
settings.receiver_config['port'])
settings.database = configure_database(settings.database_config, also_create=False)
Router.defaults(**settings.router_defaults)
Router.load(settings.handlers)
Router.RELOAD=True
view.LOADER = jinja2.Environment(
loader=jinja2.PackageLoader(settings.template_config['dir'],
settings.template_config['module']))
bq. Don't be afraid that you see this much Python, you normally wouldn't touch this file
unless it were to add your own services or to make a new version for a different configuration.
For the most part, you can just edit the @config/settings.py@ and go.
First you'll see that @config/boot.py@ sets up logging using the @config/logging.conf@
file, which you can change to reconfigure how you want logs to be created.
Then it starts assigning variables to the config.settings module that it
has imported at the top. This is important because after @config.boot@ runs
your lamson code and handlers will have access to all these services. You
can get directly to the relay, receiver, database and anything else you need
by simply doing:
from config import settings
After that @config.boot@ sets up the @settings.relay@, @settings.receiver@,
and @settings.database@. These three are used heavily in your own Lamson code,
so knowing how to change them if you need to helps you later.
After this we configure the @lamson.routing.Router@ to have your defaults,
load up your handlers, and turn on RELOAD. Setting @Router.RELOAD=True@
tell the Router to reload all the handlers for each request. Very handy when you
are doing development since you don't need to reload the server so often.
bq. If you deploy to production, then you'll want to set this to False since it's
a performance hit.
Finally, the @config.boot@ does the job os loading the template system you'll use,
in this case Jinja2. Jinja2 and Mako use the same API so you can configure
Mako here as well, as long as the object assigned to view.LOADER has the same API
it will work.
h1. Developing With Lamson
Now that you've received a thorough introduction to how to manage Lamson, and
how it is configured, you can get into actually writing some code for it.
Before you begin, you should know that writing an application for a mail server
can be a pain. The clients and servers that handle SMTP make a large number of
assumptions based on how the world was back in 1975. Everything is on defined ports
with defined command line parameters and the concept of someone pointing their
mail client at a different server arbitrarily just doesn't exist.
The world of email is not like the web where you just take any old "client" and point
it at any old server and start messing with it.
Lucky for you, Lamson has solved most of these problems and provides you with a
bunch of handy development tools and tricks so you can work with your Lamson server
without having to kill yourself in configuration hell.
h2. Using Mutt
You probably don't have another SMTP server running, and even if you did, it'd
be a pain to configure it for development purposes. You'd have to setup aliases, new
mail boxes, restart it all the time, and other annoyances.
For development, what we want is our own little private SMTP relay, and since Lamson can
also deliver mail, that is what we get with the command:
$ lamson log
This tells Lamson to run as a "logging server", which doesn't actually deliver
any mail. With this one command you have a server running on 8825 that takes every
mail it receives and saves it to the @run/queue@ Maildir and also logs it to
@logs/logger.log@. It also logs the full protocol chat to @logs/lamson.err@ so
you can inspect it.
bq. Lamson uses Maildir by default since it is the most reliable and fastest mail queue
format available. It could also store mail messages to any queue supported by Python's
"mailbox":http://docs.python.org/library/mailbox.html library. If you were
adventurous you could also use a RDBMS, but that's just silly.
You also have the file @muttrc@ which is configured to trick mutt into talking to *your*
running Lamson server, and then read mail out of the @run/queue@ maildir that is filled
in by the @lamson log@ server. Let's take a look:
set mbox_type=Maildir
set folder="run/queue"
set mask="!^\\.[^.]"
set mbox="run/queue"
set record="+.Sent"
set postponed="+.Drafts"
set spoolfile="run/queue"
set sendmail="/usr/bin/env lamson sendmail -port 8823 -host 127.0.0.1"
Notice that it's configured sendmail to be "sendmail -port 8823 -host 127.0.0.1"
which is a special @lamson sendmail@ command that knows how to talk to lamson and
read the arguments and input that mutt gives to deliver a mail.
bq. Why does Lamson need its own sendmail? Because you actually have to configure most
mail server's configuration files to change their ports before their *sendmail command*
will use a different port. Yes, the average sendmail command line tool assumes that it
is always talking to one and only one server on one and only one port for ever and all
eternity. Without @lamson sendmail@ you wouldn't be able to send to an arbitrary
server.
With this setup (@lamson start@ ; @lamson log@ ; @mutt -F muttrc@) you can now
use your mutt client as a test tool for working with your application.
h2. Stopping Lamson
The PID(Process ID) files are stored in the @run@ directory. Here's a sample
session where I stop all the running servers:
$ ls -l run/*.pid
-rw-r--r-- 1 zedshaw staff 5 May 16 16:41 run/log.pid
-rw-r--r-- 1 zedshaw staff 5 May 16 16:41 run/smtp.pid
$ lamson stop -ALL run
Stopping processes with the following PID files: ['run/log.pid', 'run/smtp.pid']
Attempting to stop lamson at pid 1693
Attempting to stop lamson at pid 1689
You can also pass other options to the stop command to just stop one server. Use
_lamson help -for stop_ to see all the options.
h2. Starting Lamson Again
Hopefully you've been paying attention and have figured out how to restart lamson and the
logging server. Just in case, here it is again:
$ lamson start
$ lamson log
You should also look in the logs/lamson.log file to see that it actually started. The
other files in the logs directory contain messages dumped to various output methods (like
Python's stdout and stderr). Periodically, if the information you want is not in
logs/lamson.log then it is probably in the other files.
bq. You can change your logging configuration by editing the logging line your config/settings.py file.
h2. Other Useful Commands
You should read the "available commands":/docs/lamson_commands.html documentation to get an
overview, and you can also use _lamson help_ to see them at any time.
h2. send
The first useful command is _lamson send_, which lets you send mail to SMTP servers (not
just Lamson) and watch the full SMTP protocol chatter. Here's a sample:
$ lamson send -port 25 -host zedshaw.com -debug 1 \
-sender tester@test.com -to zedshaw@zedshaw.com \
-subject "Hi there" -body "Test body."
send: 'ehlo zedshawscomputer.local\r\n'
reply: '502 Error: command "EHLO" not implemented\r\n'
reply: retcode (502); Msg: Error: command "EHLO" not implemented
send: 'helo zedshawcomputer.local\r\n'
reply: '250 localhost.localdomain\r\n'
reply: retcode (250); Msg: localhost.localdomain
send: 'mail FROM:\r\n'
reply: '250 Ok\r\n'
reply: retcode (250); Msg: Ok
send: 'rcpt TO:\r\n'
reply: '250 Ok\r\n'
reply: retcode (250); Msg: Ok
send: 'data\r\n'
reply: '354 End data with .\r\n'
reply: retcode (354); Msg: End data with .
data: (354, 'End data with .')
send: 'Content-Type: text/plain; charset="us-ascii"\r\nMIME-Version: 1.0\r\nContent-Transfer-Encoding: 7bit\r\nSubject: Hi there\r\nFrom: tester@test.com\r\nTo: zedshaw@zedshaw.com\r\n\r\n.\r\n'
reply: '250 Ok\r\n'
reply: retcode (250); Msg: Ok
data: (250, 'Ok')
send: 'quit\r\n'
reply: '221 Bye\r\n'
reply: retcode (221); Msg: Bye
Using this helps you debug your Lamson server by showing you the exact protocol sent
between you and the server. It is also a useful SMTP server debug command by itself.
bq. When you use the supplied muttrc you'll be configured to use Lamson's sendmail
(not *send) command as your delivery command. This lets you use mutt as a complete development
tool with minimal configuration.
h2. queue
The _lamson queue_ command lets you investigate and manipulate the run/queue (or any maildir).
You can pop a message off, get a message by its key, remove a message by its key, count the messages,clear the queue, list keys in the queue. It gives you a lower level view of the queue than
mutt would, and lets you manipulate it behind the scenes.
h2. restart
Lamson does reload the code of your project when it receives a new request (probably too
frequently), but if you change the @config/settings.py@ file then you need to restart.
Easiest way to do that is with the restart command.
h2. Walking Through The Code
You should actually know quite a lot about how to run and mess with Lamson, so you'll
want to start writing code. Before you do, go check out the "API Documentation":/docs/api/
and take a look around. This document will guide you through where everything is and how
to write your first handler, but when you start going out on your own you'll need a good
set of reference material.
At the top level of your newly minted project you have these directories:
app -- Where the application code (handlers, templates, models) lives.
config -- You already saw everything in here.
logs -- Log files get put here.
run -- Stuff that would go in a /var/run like PID files and queues.
tests -- Unit tests for handlers, templates, and models.
Lamson expects all of these directories to be right there, so don't get
fancy and think you can move them around.
The first place to look is in the app directory, which has this:
app/__init__.py
app/data -- Data you want to keep around goes here.
app/handlers -- Lamson handlers go here.
app/model -- Any type of backend ORM models or other non-handler code.
app/templates -- Email templates.
You don't technically *have* to store your data in app/data. You are free
to put it anywhere you want, it's just convenient for most situations to
have it there.
Your @app/model@ directory could have anything in it from simple modules for
working various Maildir queues, to full blown SQLAlchemy configurations for
your database. The only restriction is that you load them in the modules
yourself (no magic here).
The @app/templates@ directory can have any structure you want, and as you
saw from the @config.boot@ discussion it is just configured into the
Jinja2 configuration as the default. If you have a lot of templates it might
help to have them match your @app/handlers@ layout in some logical way.
That only leaves your @app/handlers@ directory:
app/handlers/__init__.py
app/handlers/sample.py
This is where the world gets started. If you look at your @config.settings@
you'll see this line:
handlers = ['app.handlers.sample']
Yep, that's telling the "lamson.routing.Router":http://lamsonproject.org/docs/api/lamson.routing.RoutingBase-class.html
to load your @app.handlers.sample@
module to kick it into gear. It really is as simple as just putting the file in
that directory (in in sub-modules there) and then adding them to the handlers
list.
You can also add handlers from modules outside of your @app.handlers@:
This installs the handler ("lamson.handlers.log":http://lamsonproject.org/docs/api/lamson.handlers.log-module.html) that lamson uses to log every email it receives.
h2. Writing Your Handler
This document is for getting started quickly, so going into the depths of the cool stuff
you can do with Lamson handlers is outside the scope, but if you open the _app/handlers/sample.py_
file and take a look you'll how a handler is structured.
bq. Since Lamson is changing so much the contents of the file aren't included in this document. You'll have to open it and take a look.
At the top of the file you should see your typical import statements:
import logging
from lamson.routing import route, route_like, stateless
from config.settings import relay, database
from lamson import view
Notice that we include elements from the @lamson.routing@ that are decorators
we use to configure a route. Then you'll see that we're getting that @settings.relay@
and @settings.database@ we configured in the previous sections. Finally we bring
in the @lamson.view@ module directory to make rendering templates into email messages
a lot easier.
Now take a look at the rest of the file and you'll how a handler is structured:
# Each state is a separate function in CAPS. It doesn't have to be, it just looks better.
# Above each state function is a "route":http://lamsonproject.org/docs/api/lamson.routing.route-class.html, "route_like":http://lamsonproject.org/docs/api/lamson.routing.route_like-class.html, or "stateless":http://lamsonproject.org/docs/api/lamson.routing-module.html#stateless decorator to configure how @lamson.routing.Router@ uses it.
# The "route":http://lamsonproject.org/docs/api/lamson.routing.route-class.html decorator takes a pattern and then regex keyword arguments to fill it in. The words in the pattern string are replaced in the final more complex routing regex by the keyword arguments after. However, *if you want to use regex directly you can*, "route":http://lamsonproject.org/docs/api/lamson.routing.route-class.html just needs a string that eventually becomes a regex.
# A state function changes state by returning the next function to call. You want to go to the RUNNING state, just @return RUNNING@.
# If any state function throws an error it will go into the @ERROR@ state, so if you make a state handler named ERROR it will get called on the next event and can recover.
# If you want to run a state on this event rather than wait to have it run on the next, then simple call it and return what it returns. So to have RUNNING go now, just do @return RUNNING(message, ...)@ and it will work.
# If a state has the same regex as another state, just use "route_like":http://lamsonproject.org/docs/api/lamson.routing.route_like-class.html to say that.
# If you have a "stateless":http://lamsonproject.org/docs/api/lamson.routing-module.html#stateless decorator after a "route":http://lamsonproject.org/docs/api/lamson.routing.route-class.html or "route_like":http://lamsonproject.org/docs/api/lamson.routing.route_like-class.html, then that handler will run for *all* addresses that match, not just if this handler is in that state.
That is pretty much the entire complexity of how you write a handler. You
setup routes, and return the next step in your conversation as the next
function to run. The @lamson.routing.Router@ then takes each message it receives
and runs it through a processing loop handing it to your states and handlers.
h2. How States Are Run
The best way to see how states are processed is to look at the "Router":http://lamsonproject.org/docs/api/lamson.routing.RoutingBase-class.html code that does it:
def deliver(self, message):
if self.RELOAD: self.reload()
called_count = 0
for functions, matchkw in self.match(message['to']):
to_call = []
in_state_found = False
for func in functions:
if lamson_setting(func, 'stateless'):
to_call.append(func)
elif not in_state_found and self.in_state(func, message):
to_call.append(func)
in_state_found = True
called_count += len(to_call)
for func in to_call:
if lamson_setting(func, 'nolocking'):
self.call_safely(func, message, matchkw)
else:
with self.call_lock:
self.call_safely(func, message, matchkw)
if called_count == 0:
if self.UNDELIVERABLE_QUEUE:
LOG.debug("Message to %r from %r undeliverable, putting in undeliverable queue.",
message['to'], message['from'])
self.UNDELIVERABLE_QUEUE.push(message)
else:
LOG.debug("Message to %r from %r didn't match any handlers.",
message['to'], message['from'])
What this does is take all the handlers you've loaded, and then finds which handlers have a state function that
matches the current message. It then goes through each potential match, and determines which of all the matching
state functions is "in that state". This means that, even though you have six state functions that answer to
"(list_name)-(action)@(host)" only the one that matches the users current state (say PENDING) will be called next.
As it goes through these functions it also loads up any that are marked "stateless" so they can be called as well.
Finally, it just calls them in order. If the message results in no methods to call, then it will take the message
and tell you this, or put it into an @UNDELIVERABLE_QUEUE@ for you to review it later.
bq. Slight design criticism: Currently the order of these calls is fairly deterministic, but you can't rely on it.
It's also not clear if *all* matching states should run, or just the first. It currently only runs the first match,
but it might be better to run each match from each handler. Suggestions welcome on this.
h2. Debugging Routes
In the old way of doing routing you would edit a large table of "routes" in your @config/settings.py@ file and
then that told Lamson how to run. The problem with this is it was too hard to maintain and too hard to
indicate that different states needed a different route.
The new setup is great because all your routing for each handler module is right there, and it's easy to see
what will cause a particular state function to go off.
What sucks about the new setup is that you can't find out what all the routes are doing *globally* in one
place. That's where @lamson routes@ comes in. Simply run that command and you'll get a debug dump of
all the full routing regex and the functions and modules they belong to:
This is telling you which regex is matched first, then what those regex are mapped to. This is very handy as you can copy-paste
that regex right into a python shell and then play with it to see if it would match what you want.
You can also pass in an email address to the @-test@ option and it will tell you what routes would match
and which functions that will call:
osb $ lamson routes -test test.blog@oneshotblog.com
2009-06-07 02:33:31,678 - root - INFO - Database configured to use sqlite:///app/data/main.db URL.
Routing ORDER: [... lots of regex here ...]
Routing TABLE:
---
... each regex and what state functions it maps ..
---
'^post-confirm-(?P[a-z0-9]+)@(?Poneshotblog\\.com)$': app.handlers.post.CONFIRMING
---
TEST address 'test.blog@oneshotblog.com' matches:
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.index.POSTING
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.post.START
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.post.POSTING
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
osb $
If you're working with Lamson this is incredibly helpful, because it tells you what
routes you have, what functions they call, and then it'll take an email address and
tell you all the routes that match it.
h2. THREADING!
Lamson takes a lighter approach to how it runs. It assumes that most of the
time you want lamson to keep itself sane with minimal locking, and that you
want each of your state functions to run in a thread lock that prevents others
from stepping on your operations. In 95% of the cases, this is what you want.
To accomplish this, Lamson's router will acquire an internal lock for
operations that change its state, and a separate lock before it calls each
state function. Since multiple state functions run inside each thread, but one
thread handles each message, you'll get multiple processing, but each state
won't step on other states in the system.
However, it's those 5% of the times that will kill your application, and if you
know what you're doing, you should be able to turn this off. In order to tell
the Router *not* to lock your state function, simply decorate it with
"nolocking":http://lamsonproject.org/docs/api/lamson.routing-module.html#nolocking
and Lamson will skip the locking and just run your state raw. This means that
other threads will run potentially stepping on your execution, so you *must* do
your own locking.
Now, don't think that slapping a
"nolocking":http://lamsonproject.org/docs/api/lamson.routing-module.html#nolocking
on your state functions is some magic cure for performance issues. You only
ever want to do this if you *really* know your stuff, and you know how to make
that operation faster with better controlled locking.
The reality is, if you have an operation that takes so long it blocks
everything else, then you are doing it wrong by trying to do it all in your
state function. You should change your design so that this handler drops the
message into a
"lamson.queue.Queue":http://lamsonproject.org/docs/api/lamson.queue.Queue-class.html
and that *another* Lamson server reads messages out of that to do the long
running processing.
Using queues and separate Lamson servers you can solve most of your processing
issues without a lot of thread juggling and process locking. In fact, since
Lamson uses maildir queues by default you can even spread these processors out
to multiple machines reading off a shared disk and everything will be just
fine.
But, since programmers will always want to just try turning off the locking,
Lamson supports the @nolocking@ decorator. Use with care.
h2. What's In A Unit Test
Writing unit tests is way outside the scope of this document, but you should read up on using nosetests, testunit, and
you should look at "lamson.testing":http://lamsonproject.org/docs/api/lamson.testing-module.html for a bunch of helper functions. Also look in the generated @tests@ directory
to see some examples.
h2. Spell Checking Your Email Templates
Another big help is that Lamson has support for "PyEnchant":http://www.rfk.id.au/software/pyenchant/ so you
can spell check your templates. You can use "lamson.testing.spelling":http://lamsonproject.org/docs/api/lamson.testing-module.html#spelling function in your unit tests.
Installing PyEnchant is kind of a pain, but the trick is to get the dictionary you want and put it in your
@~/.enchant/myspell@ directory. You'll also want to open the @config/testing.py@ file and uncomment the
lines at the bottom that tell PyEnchant where to find the enchant so (dylib).
PyEnchant is kind of hard to use, so if you have suggestions on a better Python spell checking lib for
unit tests please let "me know.":/contact.html
h2. Spam Filtering For Free
Lamson comes with the
"lamson.spam":http://lamsonproject.org/docs/api/lamson.spam-module.html module
which supports "SpamBayes":http://spambayes.sourceforge.net/ spam filtering
system.
Read the document on "Filtering Spam With Lamson":/docs/filtering_spam.html to get
a full set of instructions on using the spam filtering features.
h2. Other Examples
Next you'll want to sink your teeth in a bigger example. Go grab "the source distribution .tar.gz":/releases/ and
extract it so you can get at the examples:
$ tar -xzvf lamson-VERSION.tar.gz
$ cd lamson-VERSION
$ cd examples/osb
You are now in the osb example that is running on
"oneshotblog.com":http://oneshotblog.com/. Using what you've learned so far
you can start reviewing the code and finding out how a working example
operates.
h2. Getting Help
As you work through this documentation, send your questions "to me":/contact.html and I'll try to help
you. You can also join the "lamson@librelist.com mailing list":mailto:lamson@librelist.com and get help
from other Lamson users.
lamson-1.0pre11/doc/lamsonproject.org/input/docs/hooking_into_django.txt 0000644 0000765 0000024 00000005235 11243575422 026132 0 ustar zedshaw staff Title: Hooking Lamson Into Django
This is a short document because using "Django":http://www.djangoproject.com/ ORM is very simple from Lamson. The trick is
to fake out Django so that when you import the Django model into Lamson, the Django model
knows where it's living and will operate. We'll go through an "integration" step
by step.
h1. Step 1: Make Your Django App Work
There's no point in trying to import an ORM that doesn't work. So get it working,
maybe write some tests.
An important thing is that you should be able to run @python manage.py shell@ and import
your model without problems.
h2. Step 2: Fake Out Django
You then have to put an environment variable in your Lamson @config/settings.py@ file before
you load any Django Models:
This is from the "librelist.com":http://librelist.com/ examples, where the Django
models are in @webapp@ so we our settings module from Lamson perspective is @webapp.settings@.
h2. Step 3: Use The Django Models
After that, you can just import your models however you want. Here's an example of
Librelist using a Django Model to store confirmations:
h2. Well, That Wasn't Too Hard
The only thing you'll have to contend with is where code you need to use these models
lives. Since it's all Python, you can just import what you need, but my recommendation
is to focus most of your model code into your Django application, and then only put
a small amount into Lamson.
For more information, look in the "Lamson source releases":/releases/ to see how
Django is used in the @examples/librelist@ code.
lamson-1.0pre11/doc/lamsonproject.org/input/docs/html_email_generation.txt 0000644 0000765 0000024 00000014137 11243706470 026447 0 ustar zedshaw staff Title: HTML Email Generation
HTML Email is apparently the killer feature for everyone who uses email.
The first, second, and third question I get asked when I tell people
about Lamson is, "Does it do HTML email?!" Yes it does, and hopefully
in a very nice clean way that'll make blasting out all that great
marketing material your users *love* easier than ever.
bq. The fourth question is whether Lamson handles bounces. "Yes it does.":/docs/bounce_detection.html
h2. A Few Tips About HTML Email
First, you *must* educate your marketing people that your user's "Inbox is not a TV.":http://myinboxisnota.tv/
I actually run "myinboxisnota.tv":http://myinboxisnota.tv/ just to make a point that people who
do marketing have the wrong idea of people's Inbox experience. It's your job as the hacker to
educate them and make sure that they know users actually mostly hate HTML email, and that sending
them an HTML email is nothing like sending them a commercial on TV.
The next tip is a technical one related to how you have to craft your HTML. Lamson's
"lamson.html":http://lamsonproject.org/docs/api/lamson.html-module.html API actually
helps you get this right, but you need to understand it in order to appreciate what
it does for you.
The rule when crafting your HTML is that you need to code for browsers of circa 1995
with no ability to use @style@ tags or @script@ tags. Most of the various HTML viewing
clients strip or disable these making them useless. This means you have to put all
CSS stylings inline into your HTML, and you have to rely on "taboo" tags like @center@
and @table@ for your layout.
h2. The Structure Of HtmlMail
The "lamson.html.HtmlMail":http://lamsonproject.org/docs/api/lamson.html.HtmlMail-class.html class
is responsible for generating all your HTML emails. How it works is you give construct one
with an "outer template" and a "CleverCSS":http://sandbox.pocoo.org/clevercss/ template as your
CSS stylesheet. This builds a generator that you then hand an internal version of your email
to generate the content for each user.
This combination of outer template+CleverCSS and inner markdown content means that you can
use the markdown content also as your text version, since markdown actually looks half-decent
as a text format.
Here's a simple example that shows this process in action:
generator = html.HtmlMail("style.css", "html_test.html", {})
resp = generator.respond({}, "content.markdown",
From="zedshaw@zedshaw.com",
To="zedshaw@zedshaw.com",
Subject="Test of an HTML mail.",
Body="content.markdown")
First, we make a generator passing in the CleverCSS stylesheet @style.css@,
and the Jinja2 HTML template @html_test.html@. Then we call @HtmlMail.respond@
to craft a "lamson.mail.MailResponse":http://lamsonproject.org/docs/api/lamson.mail.MailResponse-class.html
that you can then hand to "lamson.server.Relay":http://lamsonproject.org/docs/api/lamson.server.Relay-class.html
for delivery.
There's also a couple of tricks in the above email. First, notice that the
second parameter is "content.markdown", but that we also pass in the Body="content.markdown".
This little convenience tells the HtmlMail object that you want to reuse the
raw @content.markdown@ file as the text/plain version of the email.
Finally, once you make the generator you can keep calling @respond@ to spit out
each message you want.
h2. The HTML CSS Conversion
None of the above code shows you what is the nicest part of this API. The lamson.html
API will actually take plain HTML and your CleverCSS and insert the @style@ attributes
into it for you. Let's look at an example from the unit test (that's disgusting).
First we have the CleverCSS template:
body:
margin: 10
padding: 20
background: green - 30
color: blue
h1:
font-size: 3em
h2:
font-size: 2em
color: yellow
h3:
font-size: 1em
p:
padding: 0.3em
background: red
h2:
color: yellow
#bright:
background: black
color: white
.dull:
background: gray
color: black
Then we have the raw original HTML:
<html>
<head>
<title>{{ title }}</title>
</head>
<body style="background: magenta">
<h1 class="bright">{{ title }}</h1>
{{ content }}
<h3 id="dull">All done.</h3>
</body>
</html>
Notice this is a template too, and that {{ content }} is your @content.markdown@ file
from the earlier discussion.
Now when you run this (including the content.markdown not shown here), Lamson produces
this:
<html>
<head>
<title></title>
</head>
<body style="background: magenta; margin: 10; padding: 20; background: #006200; color: blue">
<h1 class="bright" style="font-size: 3em; background: black; color: white"></h1>
<h1 style="font-size: 3em">Hello</h1>
<p style="padding: 0.3em; background: red">I would <em>love</em> for you to tell me what is going on here joe. NOW!</p>
<h2 style="font-size: 2em; color: yellow; color: yellow">Alright</h2>
<p style="padding: 0.3em; background: red">This is the best I can come up with.</p>
<p style="padding: 0.3em; background: red">Zed</p>
<h3 id="dull" style="font-size: 1em; background: gray; color: black">All done.</h3>
</body>
</html>
Which, if you code for a Web 2.0 company is probably making your eyes bleed Dijon mustard, but it
works. Lamson has walked your HTML and inserted all the style tags it could, including
keeping any you already had there.
h2. Conclusion
With Lamson HTML email API you should be able to blast out all the wonderful HTML you need
to prop up your sales needs for years to come. It does the best it can to make it easy
to still work in a modern web methodology, but produce the nasty HTML that has the highest
chance of working in most email clients.
lamson-1.0pre11/doc/lamsonproject.org/input/docs/index.txt 0000644 0000765 0000024 00000006277 11243570044 023232 0 ustar zedshaw staff Title: Lamson Project Documentation
This is the Lamson documentation, organized into categories from "newbie" to "expert". You should
also check out the "screen casts":/videos/ available which might help you if you're the more mediated
type.
The documentation is only updated after major releases, and it may not have the best information.
If you are following the docs and they're wrong, then "send an email to lamson@librelist.com":mailto:lamson@librelist.com
and report it.
Finally, every document is also uploaded with it's text version.
h2. Initial Concepts
* "Frequently Asked Question":/docs/faq.html -- Questions people have asked about Lamson.
* "Getting Started":/docs/getting_started.html -- A fast tour of getting Lamson going and doing something with it.
* "Lamson Commands":/docs/lamson_commands.html -- All of the commands Lamson supports. You can get at this with @lamson -help@.
* "Introduction To Finite State Machines":/docs/introduction_to_finite_state_machines.html -- Important to understand the simplified version of Finite State Machines Lamson uses.
h2. Advanced Concepts
* "Deferred Processing To Queues":/docs/deferred_processing_to_queues.html -- Very handy way of processing mail.
* "Writing A Custom State Store":/docs/writing_a_state_storage.html -- You'll need this if you want to store state in the database.
* "Primary Vs. Secondary Registration":/docs/primary_vs_secondary_registration.html -- The concept of doing registration in Lamson, where contacting the service the first time is the registration.
* "Hooking Into Django":/docs/hooking_into_django.html -- Shows you how to get access to a Django ORM model.
h2. Deployment
* "Deploying Lamson Level 0 With Virtualenv And Pip":/docs/lamson_virtual_env.html -- Quick instructions for setting up your Lamson in a virtualenv for your first simple deployment.
* "Deploying Lamson Level 1":/docs/deploying_lamson.html -- This is for when you're getting more serious about deployment. Involves building a completely separate virtualenv+python for Lamson and shows deploying oneshotblog in it.
* "Deploying Lamson Level 2":/docs/deploying_lamson_level_2.html -- At this level of deployment you are running multiple sites on the same server using a virtualhost setup, and you have each application installed under its own user.
h2. Deployment: Examples
* "Deploying OneShotBlog":/docs/deploying_oneshotblog.html
h2. Specific Features
* "Unit Testing":/docs/unit_testing.html -- Lamson has a few simple things to help write better mail specific unit tests.
* "Confirmations":/docs/confirmations.html -- Confirming a user so that you validate they are an actual email address.
* "Filtering Spam":/docs/filtering_spam.html -- How to use Lamson's spam blocking features. It's easy to use, but a bit hairy to setup.
* "Bounce Detection":/docs/bounce_detection.html -- Using Lamson's bounce message parser to handle bounces.
* "Unicode Encoding And Decoding":/docs/unicode_encoding_and_decoding.html -- How Lamson decodes the nastiest email into Unicode, and then converts Unicode back into a clean email for sending.
* "HTML Email Generation":/docs/html_email_generation.html -- Using Lamson's HTML email generation library to send out HTML to annoy everyone with.
lamson-1.0pre11/doc/lamsonproject.org/input/docs/introduction_to_finite_state_machines.txt 0000644 0000765 0000024 00000025451 11242713711 031745 0 ustar zedshaw staff Title: A Painless Introduction To Finite State Machines
Lamson uses the concept of a Finite State Machine to do the internal processing
and keep track of what it should do next. You don't necessarily need to know
the details of how a FSM works, but it helps you if you want to know enough
about Lamson to do advanced work. Most people could be blissfully unaware of
state machines and still do plenty of work with Lamson.
h2. Your Computer Science Class Sucked
When I say "finite state machine" everyone reads it as:
bq. "FINITE STATE MACHINE!!!!! OMGWTFBBQ THOSE ARE HARD!!!!"
Yes, the way your professor explained FSM makes them much harder than they are
in practice. You were probably thrown fairly random sounding terms like
"edge", "node", "transitions", "acceptors", "recognizers", and "transducers".
You were probably shown graphs with lines and circles and told confusingly that
the lines were edges and the circles were nodes, but that the edges were states
unless you used this kind where the edges were transitions and that kind where
the circles were states.
Alright, just forget about that because they aren't really that complicated. A
practical finite state machine is basically four things:
* A bunch of functions, or things that need to get done.
* A bunch of events, or reasons to call these functions.
* Some piece of data that tracks the "state" this bunch of functions is in.
* Code inside the functions that says how to "transition" or "change" into the next state for further processing.
That is really all there is to every FSM. Sure you can change around what a
state is, add code that runs on a transition, or make FSM that "inherit" from
other FSM, but in the end they are all:
bq. functions, events, states, transitions
That is all there is to it, no matter how you slice these or dice these up.
h2. Implementing A Practical FSM
Now that we've brought the fear level down, let's actually make one. This will
be a little toy FSM, but it will get you started and it won't be too far off
from what Lamson uses.
We are going to implement your classic email confirmation system. This system
usually works like this:
# You receive an email at some service, let's say users-subscribe@test.com.
# You reply to the sender with a confirmation message.
# They reply with an email to your confirmation's random address.
# You receive this, approve the handshake, and make them subscribed with a welcome email.
# They send an email, and since they are subscribed, you post it to the list.
This is a typical operation on most mailing lists, but with a state machine it
turns out to be fairly trivial to implement.
First, we need to look at the above conversation and think about what is more
clear: states the user will be in, or events that they will generate. In this
case I'd go with the states they will be in during the conversation:
* START -- Every machine has one of these.
* PENDING -- We sent the confirm, and waiting for their reply.
* POSTING -- They replied to the confirm and can now post.
Notice I gave each state a verb. You are saying what state this user is in.
"They are currently posting." You don't say what they "did". Now, sometimes
it just makes sense to do it differently, so don't be a slave to this setup.
I've just found it helps keep things consistent if you name the states after
what they are currently doing.
Now that we know what kind of states we're dealing with we need to solve the
next two parts of the puzzle: events or transitions. In an email application
I like to start with the events, as these are the email addresses and messages
they will be sending me. What we do now is for each state, list the addresses
we'll consider an "event" for that state.
* START -- (list_name)-subscribe@test.com
* PENDING -- (list_name)-confirm-(id)@test.com
* POSTING -- (list_name)@test.com
This is all the "events" we need right now, written out as email addresses.
bq. If we were to add the ability to unsubscribe, then we would add an event
for (list_name)-unsubscribe@test.com to POSTING so that we could transition
them from POSTING to say, SLEEPING.
We now have our states, and the events that each state answers. Last step is
to figure out what state "transitions" to which other state.
h2. Write The Functions
At this point, our FSM is simple enough that we could just write the functions,
and the logic of each function would dictate the transitions. However, if you
had a larger number of states and events you would want to sit down and draw a
diagram or a make a table of the transitions before you wrote some code.
To start our functions we'll just name them after the states and put the events
they handle at the top as pseudo code @event@ decorators:
@event("(list_name)-subscribe@test.com")
def START(...):
"""Initial setup of the user."""
...
@event("(list_name)-confirm-(id)@test.com")
def PENDING(...):
"""Waiting for them to confirm."""
...
@event("(list_name)@test.com")
def POSTING(...):
"""They are posting, anything they send we post."""
...
This is abbreviating the syntax quite a bit, is not functioning Python code but
it is pretty close. Instead of @event@ you would have a @route@ decorator and
it would have a few regexes. Otherwise it's about the same.
bq. We could also say that each state handles multiple events, which is what
you would do if POSTING handled the "unsubscribe" requests.
h2. Add Logic And Transitions
The final piece, and the part you'll spend the longest getting right, is
filling in the logic and making the transitions happen. How would you indicate
*where* each state should go next? Remember that each state is a simple Python
function, and that to "transition" means to change to another state. Well, we
have to tell whatever is running the state machine to run a *different*
function next time.
Easiest way to do that is for our handlers to just return the next function.
The "runner" will then take that, store it somewhere, and the next time an
event comes the runner will load the next function to run.
Here's the psuedo code to do just that:
@event("(list_name)-subscribe@test.com")
def START(...):
"""Initial setup of the user."""
send_confirmation()
return PENDING
@event("(list_name)-confirm-(id)@test.com")
def PENDING(...):
"""Waiting for them to confirm."""
if check_confirmation_is_good():
send_welcome()
return POSTING
else:
ignore_them()
@event("(list_name)@test.com")
def POSTING(...):
"""They are posting, anything they send we post."""
deliver_message_to_all()
return POSTING
Right away you can see that we change to the next state by just returning the
actual function to call next. When the next event comes in, the runner matches
it to the right function, calls it, and then sets up for the next one.
If you look at PENDING, you can see that it either returns POSTING if they
confirm correctly or it just ignores them. You could also transition to
"return ERROR" if you wanted to put them in an error state and send a different
message.
Looking at POSTING, you see that it just keeps returning itself to indicate
that it is staying there. If you had POSTING process "unsubscribes" then it
would simply do the unsubscribe confirm, and then transition to UNSUBSCRIBING.
That state function would then check the confirmation and unsubscribe them,
transitioning to something like DEAD or SLEEPING.
bq. An "optimization" in Lamson is that if a state function doesn't return
anything then it's assumed to just want to stay in that state. In this case
POSTING could have no return and it would work the same.
h2. Jump To vs. Transition
You now know pretty much everything you need to handle FSM except for a tiny
corner case. It is typically called the "epsilon transition" which basically
means "transition to that state without an explicit event". When you use this
is when your event needs to fire off the code for the *next* transition right
now, rather than waiting for the next event from the user.
In the above pseudo code this is simply done by actually calling that
transition function and returning whatever it returns. Let's say we wanted to
have PENDING also call POSTING with the original message they sent:
Notice that we changed the @return POSTING@ to a @return POSTING(...)@ which
returns the results of calling POSTING.
That's it, you now know epsilon transitions. Man, that was tough.
bq. The danger of this is if you don't have your FSM carefully mapped out,
then you could call a state that loops back to your state and you're in an
endless loop. Watch for that.
h2. Why Finite State Machines Anyway
Finite State Machines like this are very powerful because they behave in ways
that are consistent, easily debugged, and intelligent. Because the decision of
how each step in the chain of events is controlled by a constrained set of
states, events, and transitions, you can actually avoid many bugs you'd get in
regular classes and objects.
For example, if you were to receive another subscribe message while there is a
confirmation pending, then that event (email address) isn't recognize and just
ignored. You would see it in the logs, and see that your FSM stayed in the
PENDING state. If you then wanted PENDING to handle additional requests it's a
simple matter of adding that event and writing the code.
The way to think of the a FSM is it is like an object that has a white list of
functions, parameters, AND allowed values for its private data. If an FSM
tries to change to a state that doesn't exist it's an error. If it gets an
event it doesn't know it ignores it. If you try to transition wrong you see it
in the logs, or it's an error.
FSM also have the debugging advantage of showing you the history of not only
what states were called, but *why* they were called and what they did next.
You will see the entire conversation and can pinpoint exactly where it went
wrong.
However, the most important reason to use FSM is because this is how email and
asynchronous conversations work. When you have a conversation with someone
there is state involved. You don't have to start the entire conversation from
scratch at the start of each sentence. Instead you remember what the person
said and what state you are in (angry, sad, happy) and that controls what you
do next (punch them, run away, hug them) based on the events they send ("screw
you", "you're dead", "i love you").
The use of an FSM will make your Lamson applications seem like magic. They
will behave more like smart systems that just seems to know like a wizard what
should happen next, and *why* it should happen that way.
lamson-1.0pre11/doc/lamsonproject.org/input/docs/lamson_commands.txt 0000644 0000765 0000024 00000012064 11242706605 025267 0 ustar zedshaw staff Title: Available Lamson Commands
Content-Type: text/html
The following is also available by running lamson help and you can get the
help for each individual command with lamson help -for COMMAND replacing
COMMAND with one of these listed below.
The format for the printed options show default options as an actual setting,
and required options as a CAPITALIZED setting you must give. For example, in the
send command:
The options -port, -host, -debug, and -file have default settings, but -sender,
-to, -subject and -body require a STRing or EMAIL. Notice also that -file
defaults to False which you can change by just including -file (that
toggles it true).
lamson blast
Given a maildir, this command will go through each email
and blast it at your server. It does nothing to the message, so
it will be real messages hitting your server, not cleansed ones.
lamson cleanse
Uses Lamson mail cleansing and canonicalization system to take an
input maildir (or mbox) and replicate the email over into another
maildir. It's used mostly for testing and cleaning.
lamson gen
Generates various useful things for you to get you started.
lamson gen -project STR -FORCE False
lamson help
Prints out help for the commands.
lamson help
You can get help for one command with:
lamson help -for STR
lamson log
Runs a logging only server on the given hosts and port. It logs
each message it receives and also stores it to the run/queue
so that you can make sure it was received in testing.
lamson log -port 8825 -host 127.0.0.1 \
-pid ./run/log.pid -chroot False \
-chdir "." -umask False -uid False -gid False \
-FORCE False
If you specify a uid/gid then this means you want to first change to
root, set everything up, and then drop to that UID/GID combination.
This is typically so you can bind to port 25 and then become "safe"
to continue operating as a non-root user.
If you give one or the other, this it will just change to that
uid or gid without doing the priv drop operation.
lamson queue
Let's you do most of the operations available to a queue.
lamson queue (-pop | -get | -remove | -count | -clear | -keys) -name run/queue
lamson restart
Simply attempts a stop and then a start command. All options for both
apply to restart. See stop and start for options available.
lamson routes
Prints out valuable information about an application's routing configuration
after everything is loaded and ready to go. Helps debug problems with
messages not getting to your handlers. Path has the search paths you want
separated by a ':' character, and it's added to the sys.path.
lamson routes -path $PWD -- config.testing -test ""
It defaults to running your config.testing to load the routes.
If you want it to run the config.boot then give that instead:
lamson routes -- config.boot
You can also test a potential target by doing -test EMAIL.
lamson send
Sends an email to someone as a test message.
See the sendmail command for a sendmail replacement.
lamson send -port 8825 -host 127.0.0.1 -debug 1 \
-sender EMAIL -to EMAIL -subject STR -body STR -attach False'
lamson sendmail
Used as a testing sendmail replacement for use in programs
like mutt as an MTA. It reads the email to send on the stdin
and then delivers it based on the port and host settings.
lamson sendmail -port 8825 -host 127.0.0.1 -debug 0 -- [recipients]
lamson start
Runs a lamson server out of the current directory:
lamson start -pid ./run/smtp.pid -FORCE False -chroot False -chdir "." \
-umask False -uid False -gid False -boot config.boot
lamson status
Prints out status information about lamson useful for finding out if it's
running and where.
lamson status -pid ./run/smtp.pid
lamson stop
Stops a running lamson server. Give -KILL True to have it
stopped violently. The PID file is removed after the
signal is sent. Give -ALL the name of a run directory and
it will stop all pid files it finds there.
lamson stop -pid ./run/smtp.pid -KILL False -ALL False
lamson version
Prints the version of Lamson, the reporitory revision, and the
file it came from.
lamson web
Starts a very simple files only web server for easy testing of applications
that need to make some HTML files as the result of their operation.
If you need more than this then use a real web server.
lamson web -basedir "." -port 8888 -host '127.0.0.1'
This command doesn't exit so you can view the logs it prints out.
lamson-1.0pre11/doc/lamsonproject.org/input/docs/lamson_virtual_env.txt 0000600 0000765 0000024 00000003155 11242705742 026016 0 ustar zedshaw staff Title: Installing Lamson Into Virtualenv With Pip
The best way to install Lamson in a virtualenv is using the "pip
tool":http://pip.openplans.org/ to install it. This will automatically fetch
the code from bazaar, install it inside the virtualenv and fetch any missing
dependencies.
First, if you haven't already, you'll need to install pip. Make sure
you've activated the virtualenv:
. bin/activate
from the root of the virtualenv, and then run:
easy_install -U pip
If you have bazaar installed on your machine, then just do:
This'll install the development version of lamson into a src/lamson
directory (src/ will be created if it doesn't exist). It will also
install all of Lamson's dependencies into the virtualenv. If you
want to update to the latest development version of Lamson, run:
cd src/lamson
bzr pull
This will fetch the latest bazaar tip from Launchpad.
If you don't have bazaar installed and don't want to install it, you
can grab the latest release of Lamson like this:
Remember though that if you do install it like this you'll need to
manually update it by visiting Launchpad, finding the latest tarball
URL and `pip install`ing it.
h2. Credits
These instructions were donated to Lamson by "Zachary
Voase":http://disturbyte.github.com but "contact me":/contact.html for
suggested improvements.
lamson-1.0pre11/doc/lamsonproject.org/input/docs/primary_vs_secondary_registration.txt 0000644 0000765 0000024 00000015756 11242714123 031156 0 ustar zedshaw staff Title: Primary vs. Secondary Registration
When you design a Lamson application the question of "registration" comes up
almost immediately. The question is one of how you find out who a person is so
that you can interact with them. In the web world this involves a form of some
kind asking for a login and password at a minimum. Some sites even go so far
as to require a sign-up process *before* they let you see anything about the
site, or even their marketing material.
On the web this kind of registration is what I like to call "Primary
Registration". It comes from the concept of primary vs. secondary data, or
primary vs. secondary analysis. In the world of measurement a primary source
is one that you gathered the information from directly, usually by giving them
a questionaire. A secondary source is one that you gathered in some indirect
way, either by evaluating past research in a new way, data mining, or simply
unobtrusive data collection.
With a web site you *must* have a primary registration system. There is no
implied identity in a browser that is consistent for a user, so you need to ask
them for some information and associate their stateless browser to a cookie or
similar tracking mechanism. You have to do this for each session you start
with them, and to register them if your service requires some kind of password
protection.
In the world of the web primary registration is just how things are done.
h2. Secondary Registration
Lamson give you the ability to do *Secondary Registration* where the act of
interacting with your service *is* the registration. A typical primary
registrationg process will ask for a user's email address to identify the user,
but in Lamson the user actively gives your service their email address to start
playing. Their first message to your service actually has all the information
you need to sign them up and give them their first taste of your service.
In fact, a user's identity is so baked into the email protocols that your
Lamson server has almost the inverse problem: it's too easy to fake an
identity, so you need to confirm their subscription and important actions.
This confirmation is traditionally nothing more than a reply asking them to
reply with some random number in the address. If you get a reply to this
auto-generated email address then you assume they are real.
bq. This "random number confirmation handshake" is important because SMTP
allows anyone with an internet connection to lie about who they are and craft a
fake email claiming to be someone else. The confirmation assures you that the
supposed sender has at least received an email in their inbox and that it had
the random number you generated in the reply address. A determined hacker
could also get past that, but if you have that situation you are dealing with
more problems than just confirmations.
Once you do though you are fairly assured that they are identified *and* using
your service. It ends up being the most low effort entry to your service you
could possibly devise without making the service unsafe.
h2. Comparing Two Designs
When you sit down to build a Lamson application you may still think about
registration in the primary way, but a secondary registration is typically more
natural for an email service. To illustrate the point, let's say you are
creating an online game using Lamson, and you want people to register for the
game.
With a primary registration you would approach the problem by making your game
open only to registered users. If they send an email to your game you send a
single reply pointing them at the web page they can use to sign up. You also
then take every email and pass it through a filter that checks that the user is
registered in your subscription database. This is mostly how you would do it
in a web application.
bq. Maybe there is a legitimate reason to force a user to a web page to give a
password and other information before using the service. If that's the case,
then go for it, but consider trying to make the *entry* to your service a
secondary registration that then asks them to do a full registration to get
more features.
With a secondary registration system, you are using Lamson's ability to track
the state of every user to know whether they are registered or not. When your
game receives their first email, it would transition to an @UNKNOWN@ state and
reply with a confirmation email. Once they reply to the confirm email you
transition from @UNKNOWN@ to the rest of the game since they are now
registered. Every interaction after that will simply know they are registered
because they are in the right state.
This has a big impact on your game's design because now you do *not* need to
check the database on every received email to confirm their identity. Their
email address and their state in your state's storage tells you that
implicitly.
You can also extend this to other requirements for your system. You can use
their known state to transition them from free to paid quality services, to
suspended states, to pause them after a bounce, and back to normal activity.
h2. Invite Only Systems
An invite only system can also use the secondary registration technique by
having the inviter's invitation request create a record for the invitee that's
in the right start state. When the inviter tells you to invite their friend,
you create a state for the friend that says @INVITED@ and use the invite as a
sort of implied confirmation email. When the invitee replies to this invite
email, you transition them to the rest of the system as they've now registered.
Another way to think of an invite only system in Lamson is as a 3rd-party
confirmation. Rather than the target user (the invitee) prompting a
confirmation, it is their friend sending the invite (the inviter) who triggers
the confirmation process on behalf of their buddy.
An additional bonus with Lamson is that the state of the invitee is known, so
if the inviter tries to send them again, or if someone else does, then you know
they already were sent an invite. You can just ignore the request and
avoid spamming people. You can also run reports to find out your
conversion rates by simply looking for everyone in the various states and
counting.
h2. Security And Design Issues
Doing these kind of secondary registrations and implied identity does come with
an extra security penalty. Unless your system is incorporating S-MIME or GPG
you won't have a solid way to identify the sender. You don't have this on most
other internet services either so it's not much of a loss. At least with email
you have a higher assurance that this is a real person.
When you design your service it's a good idea to:
# Either make everything the user does non-destructive,
# Send confirmations for anything desctructive,
# Or bounce them to a web site for more complex interactions and security.
You should also make any confirmations you send easily replied to, but have a
good random number in them that you remember for later. In fact, if you use
the "lamson.confirm" API you get all of this for free and done the right way.
lamson-1.0pre11/doc/lamsonproject.org/input/docs/unicode_encoding_and_decoding.txt 0000644 0000765 0000024 00000014431 11243702515 030064 0 ustar zedshaw staff Title: Unicode Encoding And Decoding
The world is Unicode, but email existed long before that world. Lamson uses
special encoding/decoding gear that takes just about any nasty horrible pre-Unicode
email it gets and converts it to a perfect little Unicode fantasy. You do all
your work in the Unicode world, which Python favors, and then when you're ready
to send, Lamson intelligently figures out how to make an email that anyone can
read.
The added advantage of doing this conversion is that Lamson by default also
cleans up all the various tricks spammers use to get around spam checkers. Because
it's doing a conversion from just first principles of "everything must become Unicode"
the end result is a crystal clear piece of data you can filter. When you then
output the same message, it gets reconverted to sane easily parseable message.
Lamson is also so persistent in this conversion that it can convert nearly every
message that's valid, and the very tiny percentage it can't (less 1/1000th of a percent)
are entirely screwed up spam that should not be transmitted anyway.
bq. This gear apparently doesn't support pre-Hindic sanskrit, but then again neither
does Python.
h2. Why
Sometimes people ask why Lamson would bother converting everything to Unicode?
The reason is simple: everything you deal with now is Unicode. Either it's in
a representation like UTF-8, or it's internally a Python Unicode string. Databases,
web frameworks, ORM, network protocols, and the entire internet is Unicode.
However, there's another practical reason. If you were to start using Lamson to
process email, you would end up inventing almost the exact same gear to convert
headers and bodies to Unicode. The only difference is you would do it in a typical
hacker half-assed fashion that only solved your immediate problems, and you wouldn't
do it in a global useful way.
The Lamson encoding gear is basically what you should be making to deal with email
in a modern language.
h2. How
The "lamson.encoding":http://lamsonproject.org/docs/api/lamson.encoding-module.html module
is the main meat of this conversion magic. This code is probably the most dense
part of Lamson since it has to use various parsing tricks and heuristics to figure
out how to get every part of the message into a Unicode string.
The primary trick used is that Lamson does not trust anything it's given, and when
python fails to convert a header or body, it uses the wonderful "chardet":http://chardet.feedparser.org/
library to guess at what the encoding should be based on the contents. If this fails
then the entire data is considered suspect and the message is rejected.
This turns out to be surprisingly accurate in practice, and it even corrects some
invalid clients that fail to encode Subject lines but still place Chinese encodings
in them. Lamson will take those unencoded headers, fail to convert them to ascii,
run chardet on them to see they are actually Chinese, and then convert them accurately.
h2. The Rules
A set of axioms controls how the encoding is done:
# NO ENCODING IS TRUSTED, NO LANGUAGE IS SACRED, ALL ARE SUSPECT.
# Python wants Unicode, it will get Unicode.
# Any email that CANNOT become Unicode, CANNOT be processed by Lamson or Python.
# Email addresses are ESSENTIAL to Lamson's routing and security, and therefore will be canonicalized and properly encoded.
# Lamson will therefore try to "upgrade" all email it receives to Unicode internally, and cleaning all email addresses.
# It does this by decoding all codecs, and if the codec LIES, then it will attempt to statistically detect the codec using chardet.
# If it can't detect the codec, and the codec lies, then the email is bad.
# All text bodies and attachments are then converted to Python Unicode in the same way as the headers.
# All other attachments are converted to raw strings as-is.
Once Lamson has done this, your Python handler can now assume that all
MailRequest objects are happily Unicode enabled and ready to go. The rule is:
IF IT CANNOT BE UNICODE, THEN PYTHON CANNOT WORK WITH IT.
On the outgoing end (when you send a MailResponse), Lamson tries to create the
email it wants to receive by canonicalizing it:
# All email will be encoded in the simplest cleanest way possible without losing information.
# All headers are converted to 'ascii', and if that doesn't work, then 'utf-8'.
# All text/* attachments and bodies are converted to ascii, and if that doesn't work, 'utf-8'.
# All other attachments are left alone.
# All email addresses are normalized and encoded if they have not been already.
h2. Neat Tricks
The end result of this is that you can now take any email and then convert it to
any modern data format and send it over new protocols. For example, here's the
code from "librelist.com":http://librelist.com/ to implement the JSON dump
for the "archive browser":http://librelist.com/browser/ Javascript interface:
def json_encoding(base):
ctype, ctp = base.content_encoding['Content-Type']
cdisp, cdp = base.content_encoding['Content-Disposition']
ctype = ctype or "text/plain"
filename = ctp.get('name',None) or cdp.get('filename', None)
if ctype.startswith('text') or ctype.startswith('message'):
encoding = None
else:
encoding = "base64"
return {'filename': filename, 'type': ctype, 'disposition': cdisp,
'format': encoding}
def json_build(base):
data = {'headers': base.headers,
'body': base.body,
'encoding': json_encoding(base),
'parts': [json_build(p) for p in base.parts],
}
if data['encoding']['format'] and base.body:
data['body'] = base64.b64encode(base.body)
return data
def to_json(base):
return json.dumps(json_build(base), sort_keys=True, indent=4)
Since this code can assume that everything inside Lamson is fully Unicode, it only needs
to worry about binary items like images and encode those as base64.
h2. lamson cleanse
You can also use the @lamson cleanse@ command to take a Maildir or mbox as input, and
then convert it into a cleaned up Maildir as output. Try it on some of your spam
folders to watch the magic.
h2. Reporting Problems
If you happen to run into a message that you feel Lamson should accurately
decode, feel free to "report it":/contact.html and I'll take a look.
lamson-1.0pre11/doc/lamsonproject.org/input/docs/unit_testing.txt 0000644 0000765 0000024 00000025251 11243070347 024631 0 ustar zedshaw staff Title: Unit Testing
Lamson provides the
"lamson.testing":http://lamsonproject.org/docs/api/lamson.testing-module.html
to help with writing unit tests. It doesn't do everything for you, but does
enough that you can TDD your email interactions with pretend users. It
includes features for checking messages get delivered to queues, checking
spelling, and running things through a fake or real relay.
h2. The Log Server
The first thing you need to do for testing is to run the "log server":
$ lamson log
The log server acts as the smart relay host you've configured in your @config/settings.py@
file by default. What it does is take all emails that your Lamson application
"sends out" and redeposits them into @run/queue@. This queue directory is then also
used by @lamson.testing@ for checking that emails were sent out. When anything goes
wrong, you can look in this directory and see what is getting sent out.
bq. An alternative to this setup would be to have all of the sending/routing/relaying done
internal to the whole testing framework, similar to how @lamson.testing.RelayConversation@ works.
However, I found that this made testing that your server actually sends proper emails much
too difficult.
h2. Test Organization
Lamson organizes tests into directories that match the things you're testing:
librelist $ ls -l tests/
total 16
drwxr-xr-x 6 zedshaw staff 204 Aug 19 17:01 handlers
drwxr-xr-x 8 zedshaw staff 272 Aug 18 15:50 model
drwxr-xr-x 3 zedshaw staff 102 Aug 18 15:50 templates
In most of the projects I've only rarely used template tests, but I'll cover
them below. Model tests make sure that any @app.model@ classes work right, and
handler tests make sure that any @app.handler@ classes work. Nice and simple.
You can add any other directories you want to this, and you can also use
doctests if you want. This comes free with
"nosetests":http://somethingaboutorange.com/mrl/projects/nose/0.11.1/ and is
very handy.
h2. Handler Tests
We'll use an example from the "librelist.com":http://librelist.com/ code base
that validates that a user can subscribe, unsubscribe, and post a message to a
mailing list. This test was primarily written in a TDD style, since generally
interactions and usability testing works better
"TDD":http://en.wikipedia.org/wiki/Test-driven_development style.
First you have a common preamble of modules that you need to include:
from nose.tools import *
from lamson.testing import *
from config import settings
import time
from app.model import archive, confirmation
Right away you notice we just include everything from @nose.tools@ and
@lamson.testing@ so that we can use it directly. Yes, this violates Python
style guidelines, but practicality is more important than dogmatic slavery to
supposed standards.
After that we include the @config.settings@, and two modules from @app.model@ that we'll
use to check that everything was working.
bq. Notice that we don't include anything from @app.handlers@ directly. These tests are meant
to be from the perspective of a user interacting with the handler via emails.
Once we have that we do a little setup to clear set some common variables and clear
out some queues we'll need to check:
Most of these are just variables used in tests later, but the big one is the
@client@ variable. It's a
"lamson.testing.RouterConversation":http://lamsonproject.org/docs/api/lamson.testing.RouterConversation-class.html
class that lets you simulate delivering email to your Lamson project.
bq. There's also a
"lamson.testing.TestConversation":http://lamsonproject.org/docs/api/lamson.testing.TestConversation-class.html
class that actually uses your real @config/settings.py@ to connect to a Relay.
This isn't used so much, but is intended for running "smoke tests" against a
newly deployed server.
With that we're ready to write out first handler test:
def test_new_user_subscribes_with_invalid_name():
client.begin()
client.say('test-list@%s' % host, "I can't read!", 'noreply')
client.say('test=list@%s' % host, "I can't read!", 'noreply')
clear_queue()
client.say('unbounce@%s' % host, "I have two email addresses!")
assert not delivered('noreply')
assert not delivered('unbounce')
client.say('noreply@%s' % host, "Dumb dumb.")
assert not delivered('noreply')
This is the longest of the tests, and shows all the various things you can do
with the @lamson.testing@ gear. Here's what we're doing in order:
# Call @client.begin@ to clear out queues and state and start fresh.
# Use @client.say@ to send an email from that client to your Lamson application. Notice that you configured the RelayConversation to pretend to be one person with each email getting the same subject line.
# Use @lamson.testing.clear_queue@ when you want to make sure the queue is clean.
# Use @lamson.testing.delivered@ to check if a certain message from someone is in the queue.
With that you can do pretty much everything you need to send an email and make
sure you get proper replies.
Here's another example:
def test_new_user_subscribes():
client.begin()
msg = client.say(list_addr, "Hey I was wondering how to fix this?",
list_name + '-confirm')
client.say(msg['Reply-To'], 'Confirmed I am.', 'noreply')
clear_queue()
Notice in this example we have a fourth parameter @list_name + '-confirm'@ and
we get a @msg@ back from our call to @client.say@. This basically combines
@client.say@ with @delivered@ to do it in one shot. Very commonly, you'll want
to say something to your server and make sure you got a certain response, and
then do something with that response. This is how you do that.
We then use this '-confirm' email message to actually subscribe the fake user.
Finally, here's two more examples:
def test_existing_user_unsubscribes():
test_new_user_subscribes()
msg = client.say(list_name + "-unsubscribe@%s" % host,
"I would like to unsubscribe.", 'confirm')
client.say(msg['Reply-To'], 'Confirmed yes I want out.', 'noreply')
def test_existing_user_posts_message():
test_new_user_subscribes()
msg = client.say(list_addr, "Howdy folks, I was wondering what this is?",
list_addr)
# make sure it gets archived
assert delivered(list_addr, to_queue=queue(queue_path))
In @test_existing_user_unsubscribes@ what we do is call
@test_new_user_subscribes@ to go through that process again, and then we chain
off that to do an unsubscribe. There's really nothing new here other than that
little trick.
In @test_existing_user_posts_message@ we do the usual send a message and expect
a reply, but then we *also* make sure that this message was delivered to the
archiver queue.
Apart from those methods and techniques, there's really nothing more to doing a
handler test. The only additional thing would be using
"assert_in_state":http://lamsonproject.org/docs/api/lamson.testing-module.html#assert_in_state
to make sure that your handler is in a particular state. I'd recommend against
doing that too much in a handler test, since it will make your tests brittle.
I only do it when the state is very important, such as when checking that they
are in a SPAMMING or BOUNCING state that I need to enforce.
h2. Model Tests
There's less functionality available in @lamson.testing@ for doing your models. The theory
is that your models will be classes, modules, and ORM that you need to perform the majority
of your storage and analysis. Since has very little to do with email you probably won't
use @lamson.testing@ as much.
About the only things you might use are APIs for checking that queues get certain messages
in them, and that certain users are in certain states.
Here's a quick example from "librelist.com":http://librelist.com again that tests how
archives work:
from nose.tools import *
from lamson.testing import *
from lamson.mail import MailRequest, MailResponse
from app.model import archive, mailinglist
import simplejson as json
import shutil
queue_path = archive.store_path('test.list', 'queue')
json_path = archive.store_path('test.list', 'json')
def setup():
clear_queue(queue_path)
shutil.rmtree(json_path)
def teardown():
clear_queue(queue_path)
shutil.rmtree(json_path)
def test_archive_enqueue():
msg = MailResponse(From="zedshaw@zedshaw.com", To="test.list@librelist.com",
Subject="test message", Body="This is a test.")
archive.enqueue('test.list', msg)
assert delivered('zedshaw', to_queue=queue(queue_path))
This is the usual initial setup, and then some extras to make sure that the "JSON archives":http://librelist.com/browser/ is working. You'll notice that we hand construct various messages, call methods on the @app.model.archive@ module, and then use @delivered@ to make sure they're correctly delivered.
h2. Template Tests
Typically you really can only test that your templates are spelled right, or that your templates
render when given certain locals. I've found that automated testing of templates isn't incredibly
useful yet, so the only one I've written is from the "oneshotblog":http://oneshotblog.com/ example:
from nose.tools import *
from lamson.testing import *
from lamson import view
import os
from glob import glob
def test_spelling():
message = {}
original = {}
for path in glob("app/templates/mail/*.msg"):
template = "mail/" + os.path.basename(path)
result = view.render(locals(), template)
spelling(template, result)
This uses @lamson.testing.spelling@ to make sure that each template renders and
that it is spelled correctly. This uses
"PyEnchant":http://www.rfk.id.au/software/pyenchant/ to do the checking, which
turns out to be rather annoying. If you are interested in improving the
template testing setup, then feel free to talk about your ideas on "the lamson
mailing list":mailto:lamson@librelist.com (but bring code, talk is cheap).
h2. Conclusion
Hopefully you'll be able to develop your application using good testing techniques with
the @lamson.testing@ API. If you find additional testing patterns that could be included
then "talk about them on the lamson mailing list":mailto:lamson@librelist.com to see
if they're general enough for others.
lamson-1.0pre11/doc/lamsonproject.org/input/docs/writing_a_state_storage.txt 0000644 0000765 0000024 00000021466 11242714313 027025 0 ustar zedshaw staff Title: Writing A Lamson State Storage Backend
Earlier versions of Lamson assumed that you would use SQLAlchemy, and that you
wouldn't mind storing your state in the database using SQLAlchemy. Well,
during the 0.9 redesign it became very easy to let you store the state however
and wherever you want. In the new Lamson setup there is a bit more work to
create alternative storage, but Lamson comes with two default stores that you
can use to get started.
h2. The Default MemoryStore
When you get started with Lamson you'll definitely not want to go through the
trouble of setting up a custom store. For your first days of development,
using the default
"MemoryStorage":http://lamsonproject.org/docs/api/lamson.routing.MemoryStorage-class.html
is the way to go. After you get further in your development you want to switch
to the
"ShelveStorage":http://lamsonproject.org/docs/api/lamson.routing.ShelveStorage-class.html
to store the state in a simple Python
"shelve":http://docs.python.org/library/shelve.html store.
@MemoryStorage@ keeps the routing state in a simple dict in memory, and doesn't
provide any thread protection. Its purpose is for developer testing and unit
test runs where keeping the state between disks is more of a pain than it is
worth. Use the @MemoryStorage@ (which is the default) for your development
runs and for simple servers where the state does need to be maintained (very
rare).
Here's the code to MemoryStore for you to just look at, it's already included in Lamson:
class MemoryStorage(StateStorage):
"""
The default simplified storage for the Router to hold the states. This
should only be used in testing, as you'll lose all your contacts and their
states if your server shutsdown. It is also horribly NOT thread safe.
"""
def __init__(self):
self.states = {}
def get(self, module, sender):
key = self.key(module, sender)
try:
return self.states[key]
except KeyError:
self.set(module, sender, ROUTE_FIRST_STATE)
return ROUTE_FIRST_STATE
def set(self, module, sender, state):
key = self.key(module, sender)
self.states[key] = state
def key(self, module, sender):
return repr([module, sender])
def clear(self):
self.states.clear()
As you can see there isn't much to implement to make your own storage
for Lamson to use.
bq. Keep in mind that this is just the storage *Lamson* needs to operate,
you probably don't want to be accessing this in your own application,
and instead probably want to access the store you create yourself.
h2. The ShelveStorage
@ShelveStorage@ is used for your small deployments where you are mostly just
testing the deployment process or doing a small service. It will store your
data between runs, and is probably fast enough for most sites, but you'll want
to ditch it if you ever:
# Run more than one process that needs the state information.
# Start to store everything in a database anyway.
The code to ShelveStorage (which is *already* part of Lamson) is more complex
since it must keep the threads happy, but you should read through it to get
an idea of how a more complex state store would work:
class ShelveStorage(MemoryStorage):
"""
Uses Python's shelve to store the state of the Routers to disk rather than
in memory like with MemoryStorage. This will get you going on a small
install if you need to persist your states (most likely), but if you
have a database, you'll need to write your own StateStorage that
uses your ORM or database to store. Consider this an example.
"""
def __init__(self, database_path):
"""Database path depends on the backing library use by Python's shelve."""
self.database_path = database_path
self.lock = threading.RLock()
def get(self, module, sender):
"""
This will lock the internal thread lock, and then retrieve from the
shelf whatever module you request. If the module is not found then it
will set (atomically) to ROUTE_FIRST_STATE.
"""
with self.lock:
store = shelve.open(self.database_path)
try:
key = store[self.key(module, sender)]
except KeyError:
self.set(module, sender, ROUTE_FIRST_STATE)
key = ROUTE_FIRST_STATE
return key
def set(self, module, sender, state):
"""
Acquires the self.lock and then sets the requested state in the shelf.
"""
with self.lock:
store = shelve.open(self.database_path)
store[self.key(module, sender)] = state
store.close()
def clear(self):
"""
Primarily used in the debugging/unit testing process to make sure the
states are clear. In production this could be a bad thing.
"""
with self.lock:
store = shelve.open(self.database_path)
store.clear()
store.close()
h2. Using The ShelveStorage
You can use @ShelveStorage@ by simply adding this line to your @config/boot.py@
file just before you do anything else with the @Router@:
from lamson.routing import ShelveStorage
Router.STATE_STORE=ShelveStorage("run/states")
It actually doesn't matter currently when you do it, but it's good practice right now.
After you do that, restart lamson and it will start using the new store.
Notice that your *tests will not use this*. It's not a good idea to have tests
use @ShelveStorage@, but if you want to turn it on for a run to see what
happens, then you can modify @config/testing.py@ the same way. You could also
write a unit test that did this temporarily by putting that line in your test
case.
h2. What To Implement
If you want to implement your own then you just have to implement the methods
in
"StateStorage":http://lamsonproject.org/docs/api/lamson.routing.StateStorage-class.html
and make sure it behaves the same as MemoryStorage. Look at the code to
ShelveStorage for a moment to see what you need:
class StateStorage(object):
"""
The base storage class you need to implement for a custom storage
system.
"""
def get(self, module, sender):
"""
You must implement this so that it returns a single string
of either the state for this combination of arguments, OR
the ROUTE_FIRST_STATE setting.
"""
raise NotImplementedError("You have to implement a StateStorage.get.")
def set(self, module, sender, state):
"""
Set should take the given parameters and consistently set the state for
that combination such that when StateStorage.get is called it gives back
the same setting.
"""
raise NotImplementedError("You have to implement a StateStorage.set.")
def clear(self):
"""
This should clear ALL states, it is only used in unit testing, so you
can have it raise an exception if you want to make this safer.
"""
raise NotImplementedError("You have to implement a StateStorage.clear for unit testing to work.")
There really isn't much to it, just methods to get and set based on the module
and sender's email address. Also notice that you don't have to make it
readable in any complete sense, since Lamson doesn't do anything other than
get, set, and clear the state store (and it only clears on reloads and in
testing).
h2. Important Considerations
I am purposefully *not* telling you how to exactly implement it because I'm not
exactly sure what is needed as of the 0.9 release. I use the the
ShelveStorage, but I'd like to hear what other people have written and then
start building infrastructure to make that easier.
There are some important things to consider when you implement your storage
though:
# Make sure that the calls to all methods are thread safe, and potentially process safe.
# If you do thread locking, use the *with* statement and an RLock.
# If your storage is potentially very slow, then consider a caching scheme inside, but *write that after making it work correctly.*
# Do *NOT* be tempted to store junk in this like it is a "session". It should be lean and mean and only do state.
# Make sure you keep the key being used exactly as given. You can seriously mess up Lamson's Router if you start getting fancy.
h2. Attaching It To The Router
Your storage backend will then be attached to the lamson.routing.Router in the
same way as what you did with ShelveStorage. It really should be that simple
since the data stored in the state store is very minimal.
h2. Other Examples
If you want more examples then you can look at the @examples/librelist@ code to see
how "librelist.com":http://librelist.com/ uses "Django":http://www.djangoproject.com/
to store the state.
lamson-1.0pre11/doc/lamsonproject.org/input/download.txt 0000644 0000765 0000024 00000002413 11227340123 022761 0 ustar zedshaw staff From: Zed
Title: Downloading Lamson
You can get the Lamson *source*, read "the source releases":/releases/ documentation to
find out how.
For people who can't click the above link, and who still want the source code, do:
bzr branch lp:lamson
But really you should read the read "the source releases":/releases/ documentation to
find out how.
h2. Installing With Easy Install
Using easy_install is the easiest way to install, simply run "sudo easy_install lamson" and
it will do the work of getting the dependencies and setting everything up.
h2. Installing With setup.py
Once you get the source, you can also use setup.py to install Lamson. First make sure that
you have *Mako*, and *nose* installed:
If you refuse to use easy_install entirely then it is on you to find these projects and
install them how you see best. *Make sure you have the most recent version of all
dependencies.*
Then you can use this command to install:
$ python setup.py install
h2. Installing Into A Virtual Env
Read the instructions "for installing with virtual env":/docs/lamson_virtual_env.html in the "documentation":/docs/ section.
lamson-1.0pre11/doc/lamsonproject.org/input/home_template.html 0000644 0000765 0000024 00000012317 11230770103 024125 0 ustar zedshaw staff
LamsonProject: $title
We've all been there, mucking around in the sendmail m4 macros trying one more time to get the damn mailing list to update for the new users. Every time we say, "This sucks, I want to rewrite this stupid thing." Yet, when we're done, we simply crawl back to our caves covered in our sendmail wounds.
Lamson's goal is to put an end to the hell that is "e-mail application development". Rather than stay stuck in the 1970s, Lamson adopts modern web application framework design and uses a proven scripting language (Python).
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/input/index.txt 0000644 0000765 0000024 00000010721 11227340344 022267 0 ustar zedshaw staff From: Zed
Title: Lamson The Python Mail Server
template: input/home_template.html
Content-Type: text/html
Rather than stay stuck in the 1970's, Lamson adopts modern web application
framework design and uses a proven scripting language (Python).
Instead of hideous aliases files (that you never remember to update)
Lamson uses friendly regular expressions and routing decorators that make
configuring how email is handled trivial.
Rather than bizarre flat file "databases" and hashtable stores, Lamson uses anything
that Python does to store data. You can use all of the following to store data in Lamson:
There's so many great ways to store data in Python that Lamson doesn't make
any assumptions other than to provide a convenience function or two for configuring
SQLAlchemy (since it's kind of a pain to setup). In reality, there's so many
ORMs and storage mechanisms available to Python that you should try as many as
you can and use the one you like the most.
Lamson also supports extensive spam blocking through SpamBayes:
All within a framework that works well with existing legacy mail servers so you can
gradually get into using Lamson.
The 30 Second Introduction
If you have Python and easy_install already, then try this out:
$ easy_install lamson
$ lamson gen -project mymailserver
$ cd mymailserver
$ lamson start
$ lamson log
$ nosetests
$ lamson help -for send
$ lamson send -sender me@mydomain.com -to test@test.com \
-subject "My test." -body "Hi there." -port 8823
$ less logs/lamson.log
$ mutt -F muttrc
$ lamson stop -ALL run
You now have a working base Lamson setup ready for you to work on with everything you need installed.
Next Steps
Grab the code and you can read through the
quick start documentation. After you've gone through
that, best thing to do is read the code to the examples/osb example included in
the source distribution and read the rest of the
documentation on this site.
At any point, you can get help for all available Lamson commands with:
$ lamson help
You can get individual command help with:
$ lamson help -for start
Finally, if you really want to get started using Lamson to implement your dream email
application, but are completely lost, then you can contact me for help.
lamson-1.0pre11/doc/lamsonproject.org/input/lists/ 0000755 0000765 0000024 00000000000 11313464573 021563 5 ustar zedshaw staff lamson-1.0pre11/doc/lamsonproject.org/input/lists/index.txt 0000644 0000765 0000024 00000002111 11310754212 023413 0 ustar zedshaw staff From: Zed
Title: Lamson Project Mailing Lists
Lamson's mailing lists are hosted on "librelist.com":http://librelist.com/
which is also written in Lamson.
You can subscribe to the list by sending an email to "lamson@librelist.com":mailto:lamson@librelist.com
with your first message. *Remember that librelist posts this message, so introduce yourself.*
If you want to read the archives for the Lamson list, you can browse them
"here":http://librelist.com/browser/ in a very very basic
form (soon to improve).
You can also get the archives for the Lamson list by using rsync:
Be careful when you do this since that will get everything. It's better
to figure out which month or day you want using your web browser and
then get that directory only.
h2. IRC Channel
For people who don't want to join the mailing list but still need help
there's a #lamson irc channel on irc.freenode.org. Come by and ask
your question, and stick around since it might take people a little
while to respond.
lamson-1.0pre11/doc/lamsonproject.org/input/releases/ 0000755 0000765 0000024 00000000000 11313464573 022230 5 ustar zedshaw staff lamson-1.0pre11/doc/lamsonproject.org/input/releases/index.txt 0000644 0000765 0000024 00000001723 11313464030 024067 0 ustar zedshaw staff Title: Lamson Releases
You can get the Lamson source in various ways:
* Download it from "support site 1.0pre11 tag":http://support.lamsonproject.org/vinfo/1.0pre11 as a "Zip file":http://support.lamsonproject.org/zip/Lamson%20Project-1.0pre11.zip?uuid=1.0pre11
* Use "Fossil":http://fossil-scm.org to get the code as described on the "support site":http://support.lamsonproject.org/home
And don't forget you can install it with easy_install as well:
sudo easy_install --upgrade lamson
bq. If easy_install doesn't work, try without --upgrade, and if that doesn't work then
look in your site-packages directory and delete all the versions of Lamson you find.
h2. Releases
You can get the 1.0pre11 release source from the this site as well:
"1.0pre11 release tar.gz":/releases/lamson-1.0pre11.tar.gz
"1.0pre11 release Python Egg":/releases/lamson-1.0pre11-py2.5.egg
h2. Examples
You can get all of the examples from the source distribution.
lamson-1.0pre11/doc/lamsonproject.org/input/videos/ 0000755 0000765 0000024 00000000000 11313464573 021716 5 ustar zedshaw staff lamson-1.0pre11/doc/lamsonproject.org/input/videos/index.txt 0000644 0000765 0000024 00000004000 11213304060 023537 0 ustar zedshaw staff Title: Screencasts And Tutorials
Content-Type: text/html
This is the first screencast that walks you through the first half of the Deploying Lamson
tutorial. It is the first screencast so it's probably rough and the sound isn't so great. If you have tips and feedback on doing these
better contact me and let me know.
WARNING: Seeking seems to not work with this player using the highquality video. Just let it stream or download it.
Lamson started more than a year ago as a fun side project of mine
after a few evil experiences with sendmail and various mailing list software. I
realized that e-mail systems didn’t have anything like a modern “framework” for building
applications. Rather than replicate the mess of aliases, pipes, processes, and nasty m4
macros that currently existed, I decided to write something different.
Lamson was originally called “Son Of Sam”, but that name proved too difficult to work
with as a project name. Apparently people don’t like their software named after serial
killers (actually, that was his dog’s name).
“Lamson Tubes” is a colloquial name for Pneumatic Tubes which were used last century to
deliver mail, packages, and hazardous material to the corporate world. They are still
in use today.
Now Lamson is a fully functioning SMTP server, relay, proxy, and e-mail application framework.
It supports most RDBMS, has templates, and is easy to manage and deploy. Lamson is smarter
than most any e-mail processing system out there thanks to Python.
However, as great as Lamson is for processing email intelligently, it isn’t the best solution
for delivering mail. There is 30+ years of SMTP lore and myth stored in the code of mail
servers such as Postfix and Exim that would
take years to replicate and make efficient. Being a practical project, Lamson defers to much
more capable SMTP servers for the grunt work of getting the mail to the final recipient.
I currently use Lamson in my own work, but I’m always looking for people doing
interesting things with e-mail. Even if you’re a spammer, or someone trying to
destroy the spammers, I want to hear from you.
If you’ve got something interesting, feel free to contact me and
talk about it.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/about.txt 0000644 0000765 0000024 00000003537 11210043226 022471 0 ustar zedshaw staff From: Zed
Title: About Lamson
Lamson started more than a year ago as a fun side project of "mine":http://zedshaw.com/
after a few evil experiences with sendmail and various mailing list software. I
realized that e-mail systems didn't have anything like a modern "framework" for building
applications. Rather than replicate the mess of aliases, pipes, processes, and nasty m4
macros that currently existed, I decided to write something different.
Lamson was originally called "Son Of Sam", but that name proved too difficult to work
with as a project name. Apparently people don't like their software named after serial
killers (actually, that was his dog's name).
"Lamson Tubes" is a colloquial name for Pneumatic Tubes which were used last century to
deliver mail, packages, and hazardous material to the corporate world. They are still
in use today.
Now Lamson is a fully functioning SMTP server, relay, proxy, and e-mail application framework.
It supports most RDBMS, has templates, and is easy to manage and deploy. Lamson is smarter
than most any e-mail processing system out there thanks to Python.
However, as great as Lamson is for processing email intelligently, it isn't the best solution
for delivering mail. There is 30+ years of SMTP lore and myth stored in the code of mail
servers such as "Postfix":http://www.postfix.org/ and "Exim":http://www.exim.org/ that would
take years to replicate and make efficient. Being a practical project, Lamson defers to much
more capable SMTP servers for the grunt work of getting the mail to the final recipient.
I currently use Lamson in my own work, but I'm always looking for people doing
interesting things with e-mail. Even if you're a spammer, or someone trying to
destroy the spammers, I want to hear from you.
If you've got something interesting, feel free to "contact me":/contact.html and
talk about it.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/ 0000755 0000765 0000024 00000000000 11313464573 021551 5 ustar zedshaw staff lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-05-16.html 0000644 0000765 0000024 00000011453 11230770143 023472 0 ustar zedshaw staff
LamsonProject: Lamson Project Site Launched
Today I launched the Lamson Project site at
lamsonproject.org and started filling in the
content. Lamson is really turning into a fun and useful project, and hopefully
the site will get other people interested in it and using it.
I took the design from one of the many free web design sites and reused the same Python blog script that I use on my own site so getting this up and running was cake.
Subscribe to the RSS feed and I’ll soon be doing a blog post announcing mailing lists and other email services that I’ll host on lamsonproject.org as well as new documentation and code drops.
If you browse around you’ll find some useful documentation, but nothing complete just yet. I’m still writing most of it.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-05-16.txt 0000644 0000765 0000024 00000001460 11203540277 023345 0 ustar zedshaw staff title: Lamson Project Site Launched
Today I launched the Lamson Project site at
"lamsonproject.org":http://lamsonproject.org/ and started filling in the
content. Lamson is really turning into a fun and useful project, and hopefully
the site will get other people interested in it and using it.
I took the design from one of the many free web design sites and reused the same Python blog script that I use on "my own site":http://zedshaw.com/ so getting this up and running was cake.
Subscribe to the "RSS feed":/feed.xml and I'll soon be doing a blog post announcing mailing lists and other email services that I'll host on lamsonproject.org as well as new documentation and code drops.
If you browse around you'll find some useful documentation, but nothing complete just yet. I'm still writing most of it.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-05-18.html 0000644 0000765 0000024 00000020563 11230770143 023476 0 ustar zedshaw staff
LamsonProject: Bug Fix 0.8.4, Mailing Lists, Spam Blocking
A few announcements from my work on Lamson the last few days. I managed to fix
a bug, put Lamson to work doing Lamson’s mailing lists, and use Lamson to do
some spam blocking on my own email account.
Hopefully eating my own dogfood won’t be too painful.
Grab 0.8.4, Important Bug Fix
I wasn’t using the SMTPServer class in Python correctly, and was stopping the
whole server when a single channel had an error. It’s a one line fix and I’ve
been running it for a few days with no problems.
Please make sure that you upgrade if you installed Lamson:
Lamson is now running its very simple examples/mailinglist sample on
lists.lamsonproject.org. You can send an email to:
lamson.users-subscribe@lists.lamsonproject.org
And try it out. The software running that is also what’s available in the
examples/mailinglist sample (minus a few tweaks) so if you want to help make it
a real functioning mailing list then feel free to contribute.
I’m subscribed there, and I’ll be working on it over the next week to turn it
into a more complete and robust mailing list. An important thing I need to
implement is bounce and vacation detection. I’ll probably also take my spam
filter hacks and work them in as well.
Also, the software at lists.lamsonproject.org is an open mailing list system.
Feel free to use it at your own risk, and if it becomes popular then I may turn
it into a permanent thing.
As an open mailing list system, what it does is creates any list that doesn’t
exist when you subscribe. Think of it as lazy loading for mailing lists. No
need to fill out forms, beg for permission, alter aliases, edit text files, or
run a ton of shell commands.
Just subscribe and go.
Now, if that turns into a massive abuse vector then I’ll turn it off, but I’d
like to make it work for people since it demonstrates something simple that
Lamson can do which other mailing list systems are bad at.
Spam Filtering And Graylists In The Works
My personal email account (zedshaw@zedshaw.com) is now running
Spambayes inside of Lamson to filter spam.
Here’s the code (stripped down for show):
class SpamHandler(server.MessageHandler):
"""Uses app.spam to mark all messages it receives as Spam."""
def process(self, session, message, args):
filter = spam.Filter()
filter.train_spam(message.msg)
filter.close()
class FilterHandler(server.MessageHandler):
"""
Uses the spam filter to either queue up a message
into the run/queue for later review, or forward
it on if it is not spam.
"""
def process(self, session, message, args):
classifier = spam.Filter( )
classifier.filter(message.msg)
if message.msg['X-Spambayes-Classification'].startswith('spam'):
q = queue.Queue('run/spam')
q.push(str(message))
else:
self.relay.deliver(message)
This shows Lamson’s raw “handler style” of extension, which the Finite State Machine
stuff is built on. These are wired into the config/settings.py file routing
so that any mail that’s sent to zedshaw@zedshaw.com goes through FilterHandler
and any mail that sent to bogus addresses @zedshaw.com goes through SpamHandler.
It turns out that a lot of spam is delivered to random addresses at zedshaw.com, so
this works as an autotraining mechanism. Well, at least until some asshat figures
out he can game my spam filter.
The FilterHandler then takes messages that SpamBayes knows is spam and puts it into
a queue for later review. I periodically check that folder and pull out any false
positives for retraining.
It’s all very hacky right now, but seems to be working really well, so once I work out
the full workflow I’ll formalize this code and add it to Lamson as extra handlers
to use. I’ll also implement a simple graylisting system and include that too.
Well, enjoy the work so far and shoot me any comments you have.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-05-18.txt 0000644 0000765 0000024 00000010144 11213057130 023336 0 ustar zedshaw staff Title: Bug Fix 0.8.4, Mailing Lists, Spam Blocking
A few announcements from my work on Lamson the last few days. I managed to fix
a bug, put Lamson to work doing Lamson's mailing lists, and use Lamson to do
some spam blocking on my own email account.
Hopefully eating my own dogfood won't be too painful.
h2. Grab 0.8.4, Important Bug Fix
I wasn't using the SMTPServer class in Python correctly, and was stopping the
whole server when a single channel had an error. It's a one line fix and I've
been running it for a few days with no problems.
Please make sure that you upgrade if you installed Lamson:
sudo easy_install --upgrade lamson
Or you can "grab it from PyPI":http://pypi.python.org/pypi?name=lamson&version=0.8.4&:action=display
h2. Mailing Lists
Lamson is now running its very simple _examples/mailinglist_ sample on
lists.lamsonproject.org. You can send an email to:
lamson.users-subscribe@lists.lamsonproject.org
And try it out. The software running that is also what's available in the
_examples/mailinglist_ sample (minus a few tweaks) so if you want to help make it
a real functioning mailing list then feel free to contribute.
I'm subscribed there, and I'll be working on it over the next week to turn it
into a more complete and robust mailing list. An important thing I need to
implement is bounce and vacation detection. I'll probably also take my spam
filter hacks and work them in as well.
Also, the software at lists.lamsonproject.org is an open mailing list system.
Feel free to use it at your own risk, and if it becomes popular then I may turn
it into a permanent thing.
As an open mailing list system, what it does is creates any list that doesn't
exist when you subscribe. Think of it as lazy loading for mailing lists. No
need to fill out forms, beg for permission, alter aliases, edit text files, or
run a ton of shell commands.
Just subscribe and go.
Now, if that turns into a massive abuse vector then I'll turn it off, but I'd
like to make it work for people since it demonstrates something simple that
Lamson can do which other mailing list systems are bad at.
h2. Spam Filtering And Graylists In The Works
My personal email account (zedshaw@zedshaw.com) is now running
"Spambayes":http://spambayes.sourceforge.net/ inside of Lamson to filter spam.
Here's the code (stripped down for show):
class SpamHandler(server.MessageHandler):
"""Uses app.spam to mark all messages it receives as Spam."""
def process(self, session, message, args):
filter = spam.Filter()
filter.train_spam(message.msg)
filter.close()
class FilterHandler(server.MessageHandler):
"""
Uses the spam filter to either queue up a message
into the run/queue for later review, or forward
it on if it is not spam.
"""
def process(self, session, message, args):
classifier = spam.Filter( )
classifier.filter(message.msg)
if message.msg['X-Spambayes-Classification'].startswith('spam'):
q = queue.Queue('run/spam')
q.push(str(message))
else:
self.relay.deliver(message)
This shows Lamson's raw "handler style" of extension, which the Finite State Machine
stuff is built on. These are wired into the config/settings.py file routing
so that any mail that's sent to zedshaw@zedshaw.com goes through *FilterHandler*
and any mail that sent to bogus addresses @zedshaw.com goes through *SpamHandler*.
It turns out that a lot of spam is delivered to random addresses at zedshaw.com, so
this works as an autotraining mechanism. Well, at least until some asshat figures
out he can game my spam filter.
The FilterHandler then takes messages that SpamBayes knows is spam and puts it into
a queue for later review. I periodically check that folder and pull out any false
positives for retraining.
It's all very hacky right now, but seems to be working really well, so once I work out
the full workflow I'll formalize this code and add it to Lamson as extra handlers
to use. I'll also implement a simple graylisting system and include that too.
Well, enjoy the work so far and shoot me any comments you have.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-05-19.html 0000644 0000765 0000024 00000013056 11230770143 023476 0 ustar zedshaw staff
LamsonProject: New Site Look, Same Great Content
This is just a quick update to say thanks to Ken Keiter
for creating a new lamsonproject.org site layout and
design. The new site should be easier to read, have more breathing room, and look
easier on the eyes. It’s even got a logo:
Which I like quite a lot.
Feel free to shoot your comments about the design to me.
Two Days Of Spam Filtering
So far the new Lamson based spam blocking and filtering is working pretty well.
The majority of spam is blocked, however the success rate of
SpamBayes needs to improve. Right now I’ve
trained it with a large set of ham, and with a reasonable number of spam
messages as I encounter them. I think it’s received about a 20/1 (ham/spam)
ratio for training.
With that training it’s sorted about 50 spam, no false positives, and missed
about 15 spam as “unsure”, with about 4 as “ham”. The 4 that were classified
as ham were just one liners with a link in them. The unsure spam probably
could have just been treated as spam, but I’m being conservative while I test
it out.
Hopefully the success rate improves over the next few days. SpamBayes is
great for being a Python library I can use easily, but if it’s success rate
doesn’t get much higher then I may look at other options. Of course, there
are more important things I could be doing right now too.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-05-19.txt 0000644 0000765 0000024 00000002741 11204561532 023351 0 ustar zedshaw staff Title: New Site Look, Same Great Content
This is just a quick update to say thanks to "Ken Keiter":http://kenkeiter.com/
for creating a new "lamsonproject.org":http://lamsonproject.org/ site layout and
design. The new site should be easier to read, have more breathing room, and look
easier on the eyes. It's even got a logo:
!http://lamsonproject.org/images/lamson.png!
Which I like quite a lot.
Feel free to shoot your comments about the design to "me.":/contact.html
h2. Two Days Of Spam Filtering
So far the new Lamson based spam blocking and filtering is working pretty well.
The majority of spam is blocked, however the success rate of
"SpamBayes":http://spambayes.sourceforge.net/ needs to improve. Right now I've
trained it with a large set of ham, and with a reasonable number of spam
messages as I encounter them. I think it's received about a 20/1 (ham/spam)
ratio for training.
With that training it's sorted about 50 spam, no false positives, and missed
about 15 spam as "unsure", with about 4 as "ham". The 4 that were classified
as ham were just one liners with a link in them. The unsure spam probably
could have just been treated as spam, but I'm being conservative while I test
it out.
Hopefully the success rate improves over the next few days. SpamBayes is
great for being a Python library I can use easily, but if it's success rate
doesn't get much higher then I may look at other options. Of course, there
are more important things I could be doing right now too.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-05-20.html 0000644 0000765 0000024 00000010241 11230770143 023457 0 ustar zedshaw staff
LamsonProject: Lamson Project Ideas
I wrote a blog post about project ideas for Lamson
on my personal blog. Head on over if you’re looking for something to hack on, or just
want something to read that isn’t about the web.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-05-20.txt 0000644 0000765 0000024 00000000404 11205152004 023322 0 ustar zedshaw staff Title: Lamson Project Ideas
I wrote a "blog post about project ideas for Lamson":http://zedshaw.com/blog/2009-05-20.html
on my personal blog. Head on over if you're looking for something to hack on, or just
want something to read that isn't about the web.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-05-24.html 0000644 0000765 0000024 00000022107 11230770143 023467 0 ustar zedshaw staff
LamsonProject: Features For The 0.9 Release (Soon)
I've been hard at work cooking up the very nice new routing system, and
I must say it is rather tasty. I've gone and created a whole new
routing and state management design that uses decorators right in your
handler modules to indicate how each state will expect mail addresses.
For the 0.9 release happening tomorrow I've got a new Router setup,
spam filtering baked in nice and clean, improved test coverage, and
indirect state storage for those who don't like SQLAlchemy.
New Routing Decorators
The new routing design is very nice if I do say so myself. It should
reduce the amount of Python regex wizardry you need to learn, help set
reasonable defaults, and put your handlers and routing in one spot so
they are easier to maintain. In addition the new design eliminated many
files and flaws from the previous design.
The formatting on the @route decorator will hopefully simplify the use
of regular expressions. Rather the above route for
"(list_name)-(action)@(host)" gets translated by Lamson into
'^(?P[a-zA-Z.0-9]+)-(?P[a-zA-Z0-9]+)@(?Ptest.com)$'
which is a hair pulling monstrosity. It also is using defaults so that you
only have to indicate basic formatting of the email and leave the regexes to
the main configuration file.
This new setup also has some interesting implications in how you can use it.
For example, you can register multiple address forms for each state (UNKNOWN, END, etc.)
so that it can handle similar activity.
Spam Filtering Per State
Now that the routing is nice and generic, I could implement a decorator for
using SpamBayes to filter spam to your state functions. If you don't
want a particular state to receive spam then you just use the @spam_filter
decorator:
from lamson.routing import route, route_like
from lamson.spam import spam_filter
ham_db = "tests/sddb"
@route("(anything)@(host)", anything=".+", host=".+")
@spam_filter(ham_db, "tests/.hammierc", "run/queue")
def START(message, **kw):
print "Ham message received. Going to END."
return END
@route_like(START)
def END(message, *kw):
print "Done."
This simply attaches the @spam_filter decorator to START and then when START
runs it transitions to END. It's still a little rough since you're having to
configure the @spam_filter right there, rather than in a config/settings.py file,
but it works. Later versions will be much cleaner.
Indirect State Storage
I've also abstracted away the state storage, which will open the door to
backends in any kind of storage you want, not just SQLAlchemy. The only thing you'll
need to implement to use a different storage is a simple get/set/clear set of functionality
and then attach it to the Router class to make it use the new storage.
Currently the code is using a simple MemoryStorage class to speed up the unit test runs,
and then I'll implement the SQLAlchemy store and probably a Tokyo Tyrant store too.
Higher Test Coverage
I also worked on getting the unit test coverage up as high as I could. Since Lamson is a
server with nasty OS level things like sockets and daemons, it is difficult to fully test
in just simple unit tests. It's at about 85% right now with only the daemon and server
commands not being tested.
Finally, this new design gets rid of the old class based handlers, the Conversation style
of Finite State Machine handlers, and quite a bit of code that was design cruft from
my previous iterations. It should be much cleaner and meaner, and will have a nicer
experience for developers.
Hold On Though
The code in the bzr branch is still in a state of flux, so feel free to grab it and look, but
you probably don't want to actually do anything with it. I should have all the samples
converted to the new style and then I'll do a 0.9 release. It should happen tomorrow (May 25th).
I'll have another blog post and some documentation for the new stuff tomorrow and all next week.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-05-24.txt 0000644 0000765 0000024 00000012322 11213057130 023333 0 ustar zedshaw staff Title: Features For The 0.9 Release (Soon)
Content-Type: text/html
I've been hard at work cooking up the very nice new routing system, and
I must say it is rather tasty. I've gone and created a whole new
routing and state management design that uses decorators right in your
handler modules to indicate how each state will expect mail addresses.
For the 0.9 release happening tomorrow I've got a new Router setup,
spam filtering baked in nice and clean, improved test coverage, and
indirect state storage for those who don't like SQLAlchemy.
New Routing Decorators
The new routing design is very nice if I do say so myself. It should
reduce the amount of Python regex wizardry you need to learn, help set
reasonable defaults, and put your handlers and routing in one spot so
they are easier to maintain. In addition the new design eliminated many
files and flaws from the previous design.
The formatting on the @route decorator will hopefully simplify the use
of regular expressions. Rather the above route for
"(list_name)-(action)@(host)" gets translated by Lamson into
'^(?P[a-zA-Z.0-9]+)-(?P[a-zA-Z0-9]+)@(?Ptest.com)$'
which is a hair pulling monstrosity. It also is using defaults so that you
only have to indicate basic formatting of the email and leave the regexes to
the main configuration file.
This new setup also has some interesting implications in how you can use it.
For example, you can register multiple address forms for each state (UNKNOWN, END, etc.)
so that it can handle similar activity.
Spam Filtering Per State
Now that the routing is nice and generic, I could implement a decorator for
using SpamBayes to filter spam to your state functions. If you don't
want a particular state to receive spam then you just use the @spam_filter
decorator:
from lamson.routing import route, route_like
from lamson.spam import spam_filter
ham_db = "tests/sddb"
@route("(anything)@(host)", anything=".+", host=".+")
@spam_filter(ham_db, "tests/.hammierc", "run/queue")
def START(message, **kw):
print "Ham message received. Going to END."
return END
@route_like(START)
def END(message, *kw):
print "Done."
This simply attaches the @spam_filter decorator to START and then when START
runs it transitions to END. It's still a little rough since you're having to
configure the @spam_filter right there, rather than in a config/settings.py file,
but it works. Later versions will be much cleaner.
Indirect State Storage
I've also abstracted away the state storage, which will open the door to
backends in any kind of storage you want, not just SQLAlchemy. The only thing you'll
need to implement to use a different storage is a simple get/set/clear set of functionality
and then attach it to the Router class to make it use the new storage.
Currently the code is using a simple MemoryStorage class to speed up the unit test runs,
and then I'll implement the SQLAlchemy store and probably a Tokyo Tyrant store too.
Higher Test Coverage
I also worked on getting the unit test coverage up as high as I could. Since Lamson is a
server with nasty OS level things like sockets and daemons, it is difficult to fully test
in just simple unit tests. It's at about 85% right now with only the daemon and server
commands not being tested.
Finally, this new design gets rid of the old class based handlers, the Conversation style
of Finite State Machine handlers, and quite a bit of code that was design cruft from
my previous iterations. It should be much cleaner and meaner, and will have a nicer
experience for developers.
Hold On Though
The code in the bzr branch is still in a state of flux, so feel free to grab it and look, but
you probably don't want to actually do anything with it. I should have all the samples
converted to the new style and then I'll do a 0.9 release. It should happen tomorrow (May 25th).
I'll have another blog post and some documentation for the new stuff tomorrow and all next week.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-05-28.html 0000644 0000765 0000024 00000016341 11230770143 023476 0 ustar zedshaw staff
LamsonProject: 0.9-pre2 Up For Testing, Docs Too
First off, my apologies to everyone if your RSS reader went crazy today.
I include documentation changes in the RSS feed so that people can
easily track updates to the Lamson docs. However, that means when I’m
writing a lot of documentation it hits the feed repeatedly.
The good news though is that you can now get at the 0.9-pre2 release of
Lamson from the releases page and you can read the great
new documentation I’ve got.
I’m getting the 0.9 pre-releases out so that people can start
migrating and feeding me problems they encounter.
I’ll have a full list of changes and some documentation on
migrating to 0.9. The main things that changed are covered
in the getting started documentation,
notably how Lamson is configured and how routing works. Routing
is the biggest change of all so read through it.
I’ll have documentation on exactly how Routing works and
how to use it too coming soon.
If you started using Lamson at the 0.8.x level and need help
migrating contact me and I’ll help you out.
Migration should involve just tagging your state functions with
the right route decorators.
Mailing List Is Dead, Long Live OSB
I really didn’t like the way I did the mailinglist example, so
rather than keep it alive I just killed it and will rewrite it
after 0.9. In its place is the examples/osb application which
is a One-Shot-Blog. You send it an email and it makes a blog
page and index for it. It’s simple enough for everyone to
comprehend and I’ll be able to cover it in documentation easily.
The OSB sample has all the right stuff for you to see how all of
Lamson’s features work, so check it out and use it as the basis
for your own applications.
I want to get all the documentation done for the 0.9 release
and finish the very last feature that it needs. I’m hoping
to have that in the next few days, at least by Monday.
The only feature remaining for 0.9 is a SQLAlchemy based state
storage, and maybe a shelf based one. Lamson now stores its
state in an abstract StateStorage class that lets you store state however you need.
Right now all that’s implemented is a MemoryStorage which actually works well enough
for most applications that you just start off, and is great for
unit testing. But, there needs to be a permanent state store
support.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-05-28.txt 0000644 0000765 0000024 00000005643 11210151522 023344 0 ustar zedshaw staff Title: 0.9-pre2 Up For Testing, Docs Too
First off, my apologies to everyone if your RSS reader went crazy today.
I include documentation changes in the RSS feed so that people can
easily track updates to the Lamson docs. However, that means when I'm
writing a lot of documentation it hits the feed repeatedly.
The good news though is that you can now get at the 0.9-pre2 release of
Lamson from the "releases":/releases/ page and you can read the great
new documentation I've got.
Simply go to the "documentation":/docs/ list and go through. You'll
want to check out "how to install Lamson in a
virtualenv":/docs/lamson_virtual_env.html by Zachary Voase. You'll
also want to read through the current draft of the "getting
started":/docs/getting_started.html as it covers all the nitty gritty
things you need.
h2. Migrating To 0.9
I'm getting the 0.9 pre-releases out so that people can start
migrating and feeding me problems they encounter.
I'll have a full list of changes and some documentation on
migrating to 0.9. The main things that changed are covered
in the "getting started":/docs/getting_started.html documentation,
notably how Lamson is configured and how routing works. Routing
is the biggest change of all so read through it.
I'll have documentation on exactly how Routing works and
how to use it too coming soon.
If you started using Lamson at the 0.8.x level and need help
migrating "contact me":/contact.html and I'll help you out.
Migration should involve just tagging your state functions with
the right route decorators.
h2. Mailing List Is Dead, Long Live OSB
I really didn't like the way I did the mailinglist example, so
rather than keep it alive I just killed it and will rewrite it
after 0.9. In its place is the _examples/osb_ application which
is a One-Shot-Blog. You send it an email and it makes a blog
page and index for it. It's simple enough for everyone to
comprehend and I'll be able to cover it in documentation easily.
The OSB sample has all the right stuff for you to see how all of
Lamson's features work, so check it out and use it as the basis
for your own applications.
You can get it by "downloading the source":/releases/ and looking
in the examples/osb directory.
h2. 0.9 Probably This Weekend
I want to get all the documentation done for the 0.9 release
and finish the *very* last feature that it needs. I'm hoping
to have that in the next few days, at least by Monday.
The only feature remaining for 0.9 is a SQLAlchemy based state
storage, and maybe a shelf based one. Lamson now stores its
state in an abstract "StateStorage":/docs/api/lamson.routing.StateStorage-class.html class that lets you store state however you need.
Right now all that's implemented is a "MemoryStorage":/docs/api/lamson.routing.MemoryStorage-class.html which actually works well enough
for most applications that you just start off, and is great for
unit testing. But, there needs to be a permanent state store
support.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-05-31.html 0000644 0000765 0000024 00000012054 11230770143 023465 0 ustar zedshaw staff
LamsonProject: Lamson 0.9 Later Today
I have been working hard on the documentation and scrubbing the code for Lamson and the 0.9 release
coming out soon. The only things I feel I need to do before an official 0.9 release are:
Clean up the one-shot-blog demo application a bit more.
Write a small set of instructions on writing your own StateStorage.
Do a last final code review to check for any obvious problems.
Once I do that I’ll make 0.9 official. If you’re using 0.8.x right now, or just playing with
Lamson, please go grab the 0.9 pre-release and make sure it works.
There is one bug in that 0.9-pre3 release where the router won’t properly route of the form:
To: "First Last"
This is fairly rare, since someone would have to craft an email like this, and it would just
be ignored anyway. That bug is fixed in the current 0.9 source.
Please shoot me any bugs you find and I’ll work them into the 0.9 release.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-05-31.txt 0000644 0000765 0000024 00000002037 11210533442 023335 0 ustar zedshaw staff Title: Lamson 0.9 Later Today
I have been working hard on the documentation and scrubbing the code for Lamson and the 0.9 release
coming out soon. The only things I feel I need to do before an official 0.9 release are:
# Clean up the one-shot-blog demo application a bit more.
# Write a small set of instructions on writing your own "StateStorage.":http://lamsonproject.org/docs/api/lamson.routing.StateStorage-class.html
# Do a last final code review to check for any obvious problems.
Once I do that I'll make 0.9 official. If you're using 0.8.x right now, or just playing with
Lamson, please go grab the "0.9 pre-release":/releases/ and make sure it works.
There is one bug in that 0.9-pre3 release where the router won't properly route of the form:
To: "First Last"
This is fairly rare, since someone would have to craft an email like this, and it would just
be ignored anyway. *That bug is fixed in the current 0.9 source.*
Please shoot me any bugs you find and I'll work them into the 0.9 release.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-01.html 0000644 0000765 0000024 00000020030 11230770143 023454 0 ustar zedshaw staff
LamsonProject: Lamson 0.9 Is Out, Find My Bugs!
I just pushed Lamson 0.9 up to PyPI for everyone to grab and break. This release
features a complete redesign of the routing, state handling, templating, and a full
set of very complete documentation. Everyone who was using 0.8.x series should be able
to migrate to this version with some work, but it won’t be terribly painful (assuming
you have unit tests).
Getting 0.9
Easiest way to install is with easy_install
straight from PyPI:
Best thing to do is read the Getting Started instructions
and when you’re done with at least the 30 second introduction you’ll have your Lamson.
Try to go through the whole document if you can and send me feedback.
Big Changes
The biggest change is that the routing mechanism and internals are totally different.
Now instead of putting all your routes into the config/settings.py as a big array,
you just put them on each state function as a decorator. To help you debug these
fancy routes there’s also a new lamson routes command that will dump them in
their final state.
The biggest impact of the new design is the old school “BlahHandler” classes are
gone. Later versions of Lamson might bring back the ability to use a class as a
handler.
After that comes the removal of depending on SQLAlchemy for state storage. You
can now write your own state storage and
store things however you need to whatever you need. There is currently a simple
storage for storing in memory and for using a Python shelve dict.
Next there was a change to how Lamson boots up and configures itself giving you
more control over what it uses, and giving your handlers better access to your
gear. This change also let’s you use better unit tests that don’t require a
running lamson server, as well as giving you the old “integration test” style
tests that hit your real server.
There’s also tentative support for PyEnchant spell checking of your templates,
and a new lamson spell command to help you use it. See the getting started
document for how to configure that, but be warned that PyEnchant is kind of a
mean install.
Finally, I switched to Jinja2 templates by default, but you can use Mako fairly easily
by altering a configuration variable in the boot.py.
Look At All The Shiny Docs
In addition to redesigning the guts of Lamson so that it’s easier to work on and use,
I also spent a lot of time cranking out good documentation so people can get started
using it. Take a look in the documentation section and have a gander
at the API documentation that is generated from the source documentation.
New Example: One-Shot-Blog
In the source distribution you can grab here there’s an example
of doing a simple blog that uses just email to manage the blogger’s posts.
The example replaces the mailing list example (for now) as it is simpler and
doesn’t use a database.
Please Break It
The 0.9 release is the result of a lot of work, but I want people to break it and
criticize it to death. The goal is to have people beat up this basic functionality,
producing a series of 0.9.x releases, and then when everything is tight doing a 1.0
release.
Thanks Everyone
I’d like to thank everyone on Twitter, Hacker News, and yes, even Reddit for
their comments both good and bad. It helped me make good decisions knowing
what people wanted, so keep the feedback coming.
Thanks also to those who donated documentation, praise, or other help directly
to the project.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-01.txt 0000644 0000765 0000024 00000007375 11210662772 023356 0 ustar zedshaw staff Title: Lamson 0.9 Is Out, Find My Bugs!
I just pushed Lamson 0.9 up to PyPI for everyone to grab and break. This release
features a complete redesign of the routing, state handling, templating, and a full
set of very complete documentation. Everyone who was using 0.8.x series should be able
to migrate to this version with some work, but it won't be terribly painful (assuming
you have unit tests).
h2. Getting 0.9
Easiest way to install is with "easy_install":http://peak.telecommunity.com/DevCenter/EasyInstall
straight from PyPI:
sudo easy_install lamson
There's also instructions on "installing to a virtualenv":/docs/lamson_virtual_env.html
for those who like that.
Best thing to do is read the "Getting Started":/docs/getting_started.html instructions
and when you're done with at least the 30 second introduction you'll have your Lamson.
Try to go through the whole document if you can and send me feedback.
h2. Big Changes
The biggest change is that the routing mechanism and internals are totally different.
Now instead of putting all your routes into the config/settings.py as a big array,
you just put them on each state function as a decorator. To help you debug these
fancy routes there's also a new @lamson routes@ command that will dump them in
their final state.
The biggest impact of the new design is the old school "BlahHandler" classes are
gone. Later versions of Lamson might bring back the ability to use a class as a
handler.
After that comes the removal of depending on SQLAlchemy for state storage. You
can now "write your own state storage":/docs/writing_a_state_storage.html and
store things however you need to whatever you need. There is currently a simple
storage for storing in memory and for using a Python shelve dict.
Next there was a change to how Lamson boots up and configures itself giving you
more control over what it uses, and giving your handlers better access to your
gear. This change also let's you use better unit tests that don't require a
running lamson server, as well as giving you the old "integration test" style
tests that hit your real server.
There's also tentative support for PyEnchant spell checking of your templates,
and a new @lamson spell@ command to help you use it. See the "getting started":/docs/getting_started.html
document for how to configure that, but be warned that PyEnchant is kind of a
mean install.
Finally, I switched to Jinja2 templates by default, but you can use Mako fairly easily
by altering a configuration variable in the boot.py.
h2. Look At All The Shiny Docs
In addition to redesigning the guts of Lamson so that it's easier to work on and use,
I also spent a lot of time cranking out good documentation so people can get started
using it. Take a look in the "documentation":/docs section and have a gander
at the "API documentation":/docs/api/ that is generated from the source documentation.
h2. New Example: One-Shot-Blog
In the source distribution "you can grab here":/releases/ there's an example
of doing a simple blog that uses just email to manage the blogger's posts.
The example replaces the mailing list example (for now) as it is simpler and
doesn't use a database.
h2. Please Break It
The 0.9 release is the result of a lot of work, but I want people to break it and
criticize it to death. The goal is to have people beat up this basic functionality,
producing a series of 0.9.x releases, and then when everything is tight doing a 1.0
release.
h2. Thanks Everyone
I'd like to thank everyone on Twitter, Hacker News, and yes, even Reddit for
their comments both good and bad. It helped me make good decisions knowing
what people wanted, so keep the feedback coming.
Thanks also to those who donated documentation, praise, or other help directly
to the project.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-03-2.html 0000644 0000765 0000024 00000015206 11230770143 023626 0 ustar zedshaw staff
LamsonProject: Lamson 0.9.2, Test Coverage 97%
The 0.9.2 release is out and ready for everyone to easy_install. I spent the
day getting rid of my tech debt by boosting the Lamson test coverage to a
whopping 97%.
By mocking out things like sys.exit and sockets I’m able to run the code and
make sure those methods were called correctly. I could also throw exceptions
to make sure that exception handlers ran correctly.
Two Bugs Fixed
Interestingly enough, increasing the test coverage to 97% didn’t find any more
bugs. I did find one small bug and one obvious bug in the 0.9.1 release.
If you gave a trailing option that wasn’t a string it would throw an exception.
If you tried to start with a different boot script then it wouldn’t do it.
These are now fixed in the 0.9.2 release, so make sure you grab it
with the easiest method being:
$ sudo easy_install lamson
IRC Channel on irc.freenode.org
I’m now hanging out in the #lamson channel on irc.freenode.org. Anyone who wants to
come and chat about lamson can stop by. I should be there all the time, but I
may not respond immediately.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-03-2.txt 0000644 0000765 0000024 00000005044 11211630350 023472 0 ustar zedshaw staff Title: Lamson 0.9.2, Test Coverage 97%
The 0.9.2 release is out and ready for everyone to easy_install. I spent the
day getting rid of my tech debt by boosting the Lamson test coverage to a
whopping 97%.
I also wrote new documentation on "Running A Queue Receiver":http://lamsonproject.org/docs/deferred_processing_to_queues.html in a separate process using Lamson.
h2. Test Coverage 97%
I used the very awesome "mock":http://www.voidspace.org.uk/python/mock/ library
which let me mock out a bunch of the system calls that the "lamson.commands":http://lamsonproject.org/docs/api/lamson.commands-module.html
module accesses to do its work.
By mocking out things like @sys.exit@ and sockets I'm able to run the code and
make sure those methods were called correctly. I could also throw exceptions
to make sure that exception handlers ran correctly.
h2. Two Bugs Fixed
Interestingly enough, increasing the test coverage to 97% didn't find any more
bugs. I did find one small bug and one obvious bug in the 0.9.1 release.
# If you gave a trailing option that wasn't a string it would throw an exception.
# If you tried to start with a different boot script then it wouldn't do it.
These are now fixed in the 0.9.2 release, so make sure you "grab it":/download.html
with the easiest method being:
$ sudo easy_install lamson
h2. IRC Channel on irc.freenode.org
I'm now hanging out in the #lamson channel on irc.freenode.org. Anyone who wants to
come and chat about lamson can stop by. I should be there all the time, but I
may not respond immediately.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-03.html 0000644 0000765 0000024 00000014062 11230770144 023467 0 ustar zedshaw staff
LamsonProject: Lamson 0.9.1 Out, New Docs
I released Lamson 0.9.1 today so please grab it and test it and shoot me feedback.
The little changes are easier handling of attachments in MailRequest, a
small tweak to spam_filter, and then a bunch of test enhancements and extra
gear. The big change is a redesign to lamson.args so that it infers the
command line arguments from the lamson.commands function keyword arguments.
It’s mostly an internal change since others can’t write their own commands
(yet).
New Documentation
I ran into a bunch of folks who didn’t know that Lamson had built-in spam
filtering, so I wrote a document on Filtering Spam With
Lamson that tells you how
to use it.
Lamson’s spam filtering support is fairly simplistic, but still useful for
filtering crap out of your handlers. It’s biggest problem is the current
implementation requires you to use the (barely documented) “SpamBayes”
command line tools for things like training.
I’ve got some instructions to get you started with the most important
SpamBayes command line tool you’ll need for training.
ChangeLog
Here’s the log of what I did:
Big code review, cleanup, and test boosting.
Modified the MailRequest.body so that it works with attachments.
Updated the OSB in preparation for a redesign to make it more interesting.
Didn’t like the name 'attachments’ for getting what’s really all the parts of a message.
Implemented a method on MailRequest to return all of the attachments in a message reliably.
Setup the muttrc to use env to find the lamson command.
Updated the commands and utils so that it works with the new lamson.args setup.
Refactored lamson.args to use inspect to figure out the required defaults of a command function, thus removing the need for a…
Contributed change to muttrc so it uses lamson wherever it is installed, even virtualenv.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-03.txt 0000644 0000765 0000024 00000003623 11211415751 023342 0 ustar zedshaw staff Title: Lamson 0.9.1 Out, New Docs
I released Lamson 0.9.1 today so please "grab it and test it":/download.html and shoot me feedback.
The little changes are easier handling of attachments in MailRequest, a
small tweak to spam_filter, and then a bunch of test enhancements and extra
gear. The big change is a redesign to lamson.args so that it infers the
command line arguments from the lamson.commands function keyword arguments.
It's mostly an internal change since others can't write their own commands
(yet).
h2. New Documentation
I ran into a bunch of folks who didn't know that Lamson had built-in spam
filtering, so I wrote a document on "Filtering Spam With
Lamson":http://lamsonproject.org/docs/filtering_spam.html that tells you how
to use it.
Lamson's spam filtering support is fairly simplistic, but still useful for
filtering crap out of your handlers. It's biggest problem is the current
implementation requires you to use the (barely documented) "SpamBayes"
command line tools for things like training.
I've got some instructions to get you started with the most important
SpamBayes command line tool you'll need for training.
h2. ChangeLog
Here's the log of what I did:
* Big code review, cleanup, and test boosting.
* Modified the MailRequest.body so that it works with attachments.
* Updated the OSB in preparation for a redesign to make it more interesting.
* Didn't like the name 'attachments' for getting what's really all the parts of a message.
* Implemented a method on MailRequest to return all of the attachments in a message reliably.
* Setup the muttrc to use env to find the lamson command.
* Updated the commands and utils so that it works with the new lamson.args setup.
* Refactored lamson.args to use inspect to figure out the required defaults of a command function, thus removing the need for a...
* Contributed change to muttrc so it uses lamson wherever it is installed, even virtualenv.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-04.html 0000644 0000765 0000024 00000015431 11230770144 023471 0 ustar zedshaw staff
LamsonProject: OneShotBlog Sample (Hack) Running
I finally got off my ass and put the OneShotBlog sample up.
This is the code (plus a few little tweaks) from the sample that is in the Lamson source
running on another server. It’s using all the features of Lamson, include the new Queue Receiver
functionality. So far it’s working great considering I’ve just been hacking on it on the side to try
out the usability of the Lamson APIs and not really taken it seriously.
The concept of the oneshotblog is a little weird:
You send an email to SOMETAG@oneshotblog.com, it can have textile markup.
SOMETAG can be anything that’s a tag (chars, ints, periods, no leading periods), like: i.want.my.blog@oneshotblog.com
That is posted for you at http://oneshotblog.com/posts/YOURADDRESS/SOMETAG.html
I do block attempts to use a bad path by whitelisting. If you send something messed up for the tag it’s ignored.
Then your post is summarized and tossed on the front page with the most recent 50 shown.
This is done by a deferred queue receiver that indexes messages as it receives them.
If you send to that same tag, it replaces it with the new contents (although the index will show the old stuff for a while).
There’s some dumb bugs in it currently, like delete doesn’t remove your message from the index and other
fun things, but so far it’s running as expected given the work I put into it.
Anyway, feel free to try it out and shoot me feedback. Try sending really broken email too so that
I can see what Lamson does with it.
Future Features Of OSB
Just for fun I’m gonna clean up OSB and make it official. I’ll add features like commenting on a post via email, each
post is a little mailing list, and handling image and code attachments. For the code attachments I’ll colorize them
with Pygments and put them up with the post.
Mailing Lists and Failmail
I’ll be bringing back the mailing list example soon, and I’ll also make an address failmail@lamsonproject.org.
More on that later.
Python 2.6 Support
If you’re running Python 2.6 then you’ll want to wait for the 0.9.3 release later today. The deployed OSB sample
at oneshotblog.com is running on Python 2.6, so I had to fix about 4 failing tests
regarding unicode issues and weirdness with python-daemonize.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-04.txt 0000644 0000765 0000024 00000004626 11211760244 023347 0 ustar zedshaw staff Title: OneShotBlog Sample (Hack) Running
I *finally* got off my ass and put the "OneShotBlog":http://oneshotblog.com/ sample up.
This is the code (plus a few little tweaks) from the sample that is in the "Lamson source":/releases/
running on another server. It's using all the features of Lamson, include the new "Queue Receiver":/docs/deferred_processing_to_queues.html
functionality. So far it's working great considering I've just been hacking on it on the side to try
out the usability of the Lamson "APIs":/docs/api/ and not really taken it seriously.
The concept of the oneshotblog is a little weird:
# You send an email to SOMETAG@oneshotblog.com, it can have textile markup.
## SOMETAG can be anything that's a tag (chars, ints, periods, no leading periods), like: i.want.my.blog@oneshotblog.com
# That is posted for you at http://oneshotblog.com/posts/YOURADDRESS/SOMETAG.html
## I *do* block attempts to use a bad path by whitelisting. If you send something messed up for the tag it's ignored.
# Then your post is summarized and tossed on the "front page":http://oneshotblog.com/ with the most recent 50 shown.
## This is done by a deferred queue receiver that indexes messages as it receives them.
# If you send to that same tag, it replaces it with the new contents (although the index will show the old stuff for a while).
There's some dumb bugs in it currently, like delete doesn't remove your message from the index and other
fun things, but so far it's running as expected given the work I put into it.
Anyway, feel free to try it out and shoot me feedback. Try sending really broken email too so that
I can see what Lamson does with it.
h2. Future Features Of OSB
Just for fun I'm gonna clean up OSB and make it official. I'll add features like commenting on a post via email, each
post is a little mailing list, and handling image and code attachments. For the code attachments I'll colorize them
with Pygments and put them up with the post.
h2. Mailing Lists and Failmail
I'll be bringing back the mailing list example soon, and I'll also make an address failmail@lamsonproject.org.
More on that later.
h2. Python 2.6 Support
If you're running Python 2.6 then you'll want to wait for the 0.9.3 release later today. The deployed OSB sample
at "oneshotblog.com":http://oneshotblog.com/ is running on Python 2.6, so I had to fix about 4 failing tests
regarding unicode issues and weirdness with python-daemonize.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-06.html 0000644 0000765 0000024 00000032014 11230770144 023467 0 ustar zedshaw staff
LamsonProject: Lamson 0.9.3 Is Out And Sexy As Hell
This release is the result of me working on my little
oneshotblog.com project while tweaking and refining
Lamson as I go. The end result is 0.9.3 didn’t have a lot of big code changes,
but all the tiny little changes add up to a very nice release. The highlights
of this release are more secure server runs, better character encoding handling
for headers, various cleanups in how mail is queued, and fixes for Python 2.6
support.
My plan is to keep using this release, improving what is there currently and
only adding minimal features, and just making what is there tighter. The goal
is to have a 1.0 release out once I’ve used it on
oneshotblog.com and the new lamsonproject.org
mailing lists application.
The docs may be out of date with this release, but I’m going through docs tomorrow.
NYLUG Python Workshop Tuesday June 09 6:00PM-8:00PM
I’ll be presenting Lamson at the NYLUG Python Workshop Tuesday June 09 6:00PM-8:00PM
and helping people get started with it. If you wanted to try it but didn’t want
to waste the time then come down and I’ll hook you up.
OneShotBlog Sample Is Live
Lamson includes the source to the oneshotblog.com project
in the examples/osb directory. You can grab the source tar.gz
and look in there fore the source. You should also try out the sample site to help
test out Lamson, especially if you use a weird character set (since that’s the biggest
weakness currently).
This release of OSB features a bunch of bug fixes, tons of tests that improve the code
coverage, the ability to comment on a post by sending an email, decent spam filtering
of the comments, ability to flag a comment as spam, and lots of little tweaks. It
uses all of the features Lamson has right now.
It still has some work to do, and I want to add the ability to send pictures and source
code (with syntax highlight).
The ChangeLog
The change log for this release is (with the big ones bold):
Cleaned up some stray printing and logging.
Now prints a nicer message if args get a bad argument.
Managed to get a file object out of the maildir which removes the need to reconvert from a message.
A few minor test suite tweaks to get coverage on the most recent features.
Implemented a reorg of how the servers are started so that they can bind port 25 and then drop priv to a safer user.
Initial support for an undeliverable queue and a forwarding handler that you can run in a queue receiver to deliver unhandled mail to the relay. Needs tests.
Using the new Router.UNDELIVERABLE_QUEUE feature to have bad mail go to a queue.
Fixed the queue_command so that it prints what queue it is working with.
Added a setting for the Router to tell it to dump undeliverable mail to a queue.
First cut at semi-automated header encoding conversion.
Initial support for sort-of-automated header decoding/encoding on MailRequest.
Found a bug in clear_queue which made it not able to clear alternate queues.
Small changes to fix a non-idiomatic has_key usage.
Tweaked testing so that it resets the Router state when you do a client.begin() in unit tests.
Tweaked the boot vs. test logging setup to make it nicer for both situations.
Since the Router is the primary code loading mechanism, it now uses its own separate logger that you can redirect to stderr to see loading errors during dev.
Created a disconnected set of config testing gear so that it can test without relying on any examples. Also configures the logger using a config file and cleans up test output.
Added a feature to the routes command to print out how each matching regex will match a given test address.
Added an exception handler for loading and reloading handlers so that you can see which ones are broke during development.
Added a -test feature to lamson routes that let’s you test an address to see what routes it matches.
Updated for Python 2.6 so the tests will run, not sure why they pass in Python 2.5 actually.
Modification to the mail parsing so that it forces 'ascii’ encoding to keep email.message_from happy in Python 2.6
Using The Drop Privilege
Lamson can now drop privilege after binding port 25. Previously it was too difficult
to make the python-daemonize library do it correctly, but after I reorganized some code
I was able to do the privilege drop manually to implement it. What this means is you can
run your email server and stop being root right after you bind port 25.
You technically don’t have to fix the permissions, but if you use queue receivers
then you’ll need to change the permissions back. Lamson still creates a few files as root before
dropping to your given UID and GID, but hopefully it will stop doing this soon.
Obviously, you might want to know what your UID and GID is, but it’s important that you know
what Python thinks it is:
Python 2.5.1 (r251:54863, Jan 13 2009, 10:26:13)
[GCC 4.0.1 (Apple Inc. build 5465)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import os
>>> os.getgid()
500
>>> os.getuid()
500
>>>
There’s other ways to find it out, but that’s the Python way.
Using lamson routes
The lamson routes command has improved a bit so that you can inspect your routing and
test out an email address to see how it would be sorted. Here’s a sample run:
osb $ lamson routes -test test.blog@oneshotblog.com
2009-06-07 02:33:31,678 - root - INFO - Database configured to use sqlite:///app/data/main.db URL.
Routing ORDER: [... lots of regex here ...]
Routing TABLE:
---
... each regex and what state functions it maps ..
---
'^post-confirm-(?P[a-z0-9]+)@(?Poneshotblog\\.com)$': app.handlers.post.CONFIRMING
---
TEST address 'test.blog@oneshotblog.com' matches:
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.index.POSTING
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.post.START
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.post.POSTING
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
osb $
If you’re working with Lamson this is incredibly helpful, because it tells you what
routes you have, what functions they call, and then it’ll take an email address and
tell you all the routes that match it.
The output needs to be cleaned up, but for debugging this is nice.
The lamson web Command
While working on oneshotblog.com I kept having to
open up the site that it generates, so I just threw a dirt simple HTTP server
into Lamson. Here’s a sample of running it:
~ $ cd projects/lamson/examples/osb/
osb $ lamson web -basedir app/data/
Starting server on 127.0.0.1:8888 out of directory 'app/data/'
localhost - - [06/Jun/2009 12:31:11] "GET / HTTP/1.1" 200 -
localhost - - [06/Jun/2009 12:31:11] "GET /styles/reset.css HTTP/1.1" 200 -
localhost - - [06/Jun/2009 12:31:11] "GET /styles/main.css HTTP/1.1" 200 -
localhost - - [06/Jun/2009 12:31:18] code 404, message File not found
It just stays there running and printing to the console, so I just use
GNU screen and leave it
running in one of the windows.
As Usual Test And Report
Test this release out, and shoot me feedback. I’m hanging on irc.freenode.org
again in the #lamson room, so just stop in if you need help.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-06.txt 0000644 0000765 0000024 00000020043 11213057130 023333 0 ustar zedshaw staff Title: Lamson 0.9.3 Is Out And Sexy As Hell
This release is the result of me working on my little
"oneshotblog.com":http://oneshotblog.com/ project while tweaking and refining
Lamson as I go. The end result is 0.9.3 didn't have a lot of big code changes,
but all the tiny little changes add up to a very nice release. The highlights
of this release are more secure server runs, better character encoding handling
for headers, various cleanups in how mail is queued, and fixes for Python 2.6
support.
My plan is to keep using this release, improving what is there currently and
only adding minimal features, and just making what is there tighter. The goal
is to *have a 1.0 release out* once I've used it on
"oneshotblog.com":http://oneshotblog.com/ and the new "lamsonproject.org
mailing lists":/lists/ application.
h2. Getting It (As Usual)
All you need to do to get lamson is read the "download instruction":/download.html
and then go through the "getting started":/docs/getting_started.html to try it out.
The *docs may be out of date with this release*, but I'm going through docs tomorrow.
h2. NYLUG Python Workshop Tuesday June 09 6:00PM-8:00PM
I'll be presenting Lamson at the "NYLUG Python Workshop Tuesday June 09 6:00PM-8:00PM":http://nylug.org/pipermail/nylug-announce/2009-June/000795.html
and helping people get started with it. If you wanted to try it but didn't want
to waste the time then come down and I'll hook you up.
h2. OneShotBlog Sample Is Live
Lamson includes the source to the "oneshotblog.com":http://oneshotblog.com/ project
in the @examples/osb@ directory. You can "grab the source tar.gz":/releases/
and look in there fore the source. You should also try out the sample site to help
test out Lamson, especially if you use a weird character set (since that's the biggest
weakness currently).
This release of OSB features a bunch of bug fixes, tons of tests that improve the code
coverage, the ability to comment on a post by sending an email, decent spam filtering
of the comments, ability to flag a comment as spam, and lots of little tweaks. It
uses *all* of the features Lamson has right now.
It still has some work to do, and I want to add the ability to send pictures and source
code (with syntax highlight).
h2. The ChangeLog
The change log for this release is (with the big ones bold):
* Cleaned up some stray printing and logging.
* Now prints a nicer message if args get a bad argument.
* *Managed to get a file object out of the maildir which removes the need to reconvert from a message.*
* A few minor test suite tweaks to get coverage on the most recent features.
* *Implemented a reorg of how the servers are started so that they can bind port 25 and then drop priv to a safer user.*
* *Initial support for an undeliverable queue and a forwarding handler that you can run in a queue receiver to deliver unhandled mail to the relay. Needs tests.*
* Using the new Router.UNDELIVERABLE_QUEUE feature to have bad mail go to a queue.
* Fixed the queue_command so that it prints what queue it is working with.
* Added a setting for the Router to tell it to dump undeliverable mail to a queue.
* *First cut at semi-automated header encoding conversion.*
* Initial support for sort-of-automated header decoding/encoding on MailRequest.
* Found a bug in clear_queue which made it not able to clear alternate queues.
* Small changes to fix a non-idiomatic has_key usage.
* Tweaked testing so that it resets the Router state when you do a client.begin() in unit tests.
* Tweaked the boot vs. test logging setup to make it nicer for both situations.
* *Since the Router is the primary code loading mechanism, it now uses its own separate logger that you can redirect to stderr to see loading errors during dev.*
* *Created a disconnected set of config testing gear so that it can test without relying on any examples. Also configures the logger using a config file and cleans up test output.*
* *Added a feature to the routes command to print out how each matching regex will match a given test address.*
* Added an exception handler for loading and reloading handlers so that you can see which ones are broke during development.
* Added a -test feature to lamson routes that let's you test an address to see what routes it matches.
* *Updated for Python 2.6 so the tests will run, not sure why they pass in Python 2.5 actually.*
* Modification to the mail parsing so that it forces 'ascii' encoding to keep email.message_from happy in Python 2.6
h2. Using The Drop Privilege
Lamson can now drop privilege after binding port 25. Previously it was too difficult
to make the python-daemonize library do it correctly, but after I reorganized some code
I was able to do the privilege drop manually to implement it. What this means is you can
run your email server and *stop being root* right after you bind port 25.
To do this, you'll just run your server with:
You technically don't have to fix the permissions, but if you use "queue receivers":/docs/deferred_processing_to_queues.html
then you'll need to change the permissions back. Lamson still creates a few files as root before
dropping to your given UID and GID, but hopefully it will stop doing this soon.
Obviously, you might want to know what your UID and GID is, but it's important that you know
what *Python* thinks it is:
Python 2.5.1 (r251:54863, Jan 13 2009, 10:26:13)
[GCC 4.0.1 (Apple Inc. build 5465)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import os
>>> os.getgid()
500
>>> os.getuid()
500
>>>
There's other ways to find it out, but that's the Python way.
h2. Using lamson routes
The @lamson routes@ command has improved a bit so that you can inspect your routing and
test out an email address to see how it would be sorted. Here's a sample run:
osb $ lamson routes -test test.blog@oneshotblog.com
2009-06-07 02:33:31,678 - root - INFO - Database configured to use sqlite:///app/data/main.db URL.
Routing ORDER: [... lots of regex here ...]
Routing TABLE:
---
... each regex and what state functions it maps ..
---
'^post-confirm-(?P[a-z0-9]+)@(?Poneshotblog\\.com)$': app.handlers.post.CONFIRMING
---
TEST address 'test.blog@oneshotblog.com' matches:
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.index.POSTING
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.post.START
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.post.POSTING
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
osb $
If you're working with Lamson this is incredibly helpful, because it tells you what
routes you have, what functions they call, and then it'll take an email address and
tell you all the routes that match it.
The output needs to be cleaned up, but for debugging this is nice.
h2. The lamson web Command
While working on "oneshotblog.com":http://oneshotblog.com/ I kept having to
open up the site that it generates, so I just threw a dirt simple HTTP server
into Lamson. Here's a sample of running it:
~ $ cd projects/lamson/examples/osb/
osb $ lamson web -basedir app/data/
Starting server on 127.0.0.1:8888 out of directory 'app/data/'
localhost - - [06/Jun/2009 12:31:11] "GET / HTTP/1.1" 200 -
localhost - - [06/Jun/2009 12:31:11] "GET /styles/reset.css HTTP/1.1" 200 -
localhost - - [06/Jun/2009 12:31:11] "GET /styles/main.css HTTP/1.1" 200 -
localhost - - [06/Jun/2009 12:31:18] code 404, message File not found
It just stays there running and printing to the console, so I just use
"GNU screen":http://www.gnu.org/software/screen/screen.html and leave it
running in one of the windows.
h2. As Usual Test And Report
Test this release out, and shoot me feedback. I'm hanging on irc.freenode.org
again in the #lamson room, so just stop in if you need help.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-08.html 0000644 0000765 0000024 00000013533 11230770144 023476 0 ustar zedshaw staff
LamsonProject: A Screencast And Docs On Deploying Lamson And OneShotBlog
A Screencast And Docs On Deploying Lamson And OneShotBlog
I just finished writing some new documentation on Deploying Lamson and OneShotBlog that
shows you how to install Lamson and all required software into a completely clean Python 2.6 and virtualenv configuration.
The instructions take you from nothing to a running oneshotblog.com installation that you can
play with and hack on. The process of doing Lamson deployments is still a little too rough for me, but these instructions
should get people started.
Being my first screencast I can say that it is very tedious. I’ll be working on various strategies to make the
process more streamlined and easier since having to take a whole day for each one will drive me insane.
If you have suggestions on making the video better, let me know and also if you know of tools to
help clean up video. One thing I know right away needs to improve is the sound quality. An apartment in NYC is just
too damn noisy for this kind of thing.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-08.txt 0000644 0000765 0000024 00000003355 11213316150 023344 0 ustar zedshaw staff Title: A Screencast And Docs On Deploying Lamson And OneShotBlog
I just finished writing some new documentation on "Deploying Lamson and OneShotBlog":/docs/deploying_lamson.html that
shows you how to install Lamson and all required software into a completely clean Python 2.6 and virtualenv configuration.
The instructions take you from nothing to a running "oneshotblog.com":http://oneshotblog.com/ installation that you can
play with and hack on. The process of doing Lamson deployments is still a little too rough for me, but these instructions
should get people started.
I also created the first screencast "teaching you how to do the same deployment":/videos/ but stopping at installing
"oneshotblog.com":http://oneshotblog.com to keep things short. It is basically me talking and typing into a terminal
window to 12 minutes showing you how to do it. You can also "download the video":http://www.archive.org/details/Lamsonproject.orgDeployLamsonInAVirtualenv
from "archive.org":http://archive.org/ to get a better viewing experience.
Being my first screencast I can say that it is very tedious. I'll be working on various strategies to make the
process more streamlined and easier since having to take a whole day for each one will drive me insane.
If you have suggestions on making the video better, "let me know":/contact.html and also if you know of tools to
help clean up video. One thing I know right away needs to improve is the sound quality. An apartment in NYC is just
too damn noisy for this kind of thing.
Enjoy, and don't forget that I'll be "presenting Lamson at the NYLUG Python workshop tomorrow June 9th at 6:00pm":http://nylug.org/pipermail/nylug-announce/2009-June/000795.html
so come on down if you want to get your Lamson working.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-09.html 0000644 0000765 0000024 00000011020 11230770144 023464 0 ustar zedshaw staff
LamsonProject: Lamson At NYLUG Python Workshop Today @ 6:00PM
Just a quick reminder that I’ll be presenting Lamson to the
NYLUG Python Workshop today at 6:00PM. The event is at
the NY Public Library Hudson Park Branch, 66 Leroy St., NY NY 10014
in NYC and you can find out more here.
I’ll be showing people how to set up Lamson, get it installed,
do cool stuff with it, and then just answering questions and helping
people work with it.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-09.txt 0000644 0000765 0000024 00000001020 11213542615 023337 0 ustar zedshaw staff Title: Lamson At NYLUG Python Workshop Today @ 6:00PM
Just a quick reminder that I'll be presenting "Lamson":http://lamsonproject.org/ to the
NYLUG Python Workshop today at 6:00PM. The event is at
the NY Public Library Hudson Park Branch, 66 Leroy St., NY NY 10014
in NYC and you can "find out more here.":http://nylug.org/pipermail/nylug-announce/2009-June/000795.html
I'll be showing people how to set up Lamson, get it installed,
do cool stuff with it, and then just answering questions and helping
people work with it.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-14.html 0000644 0000765 0000024 00000032727 11230770144 023501 0 ustar zedshaw staff
LamsonProject: The Mailocalypse Is Upon Us!
I’m currently polishing off the two final features before I start going for
the Lamson 1.0 release. I’ve been using Lamson to make a few little cute applications
and create one thing for a potential client, and so far I haven’t had to change much
since 0.9.3. It’s great so far and I hope that Lamson 1.0 will be a fun release.
The big thing that must improve though is handling character encodings in emails.
After spending a week or more trying to come up with an automated conversion scheme that
would honor all the encodings on the planet, I had a realization.
Why Isn’t All Mail UTF-8?
Remembering the purpose of Lamson as a modern mail server and framework, it finally
struck me as dumb that I’m trying to keep around encodings that were invented in the
pre-Unicode days. Every modern system understands and displays UTF-8, and apart from
some pissy attitudes from the Japanese about the Han unification, there’s really no reason
that every email Lamson handles can’t be “upgraded” to UTF-8.
As I thought about this more I started to like the idea. Take all mail Lamson handles, and
upgrade it to utf-8. Then inside Lamson you know that it’s future proofed and ready to
be used by Python no matter what. Finally when the mail goes out, the conversion for sending
is simply to encode the email as proper UTF-8 encoding consistently.
However, I’m not the type of person to think any random idea I have is instantly valid because
I thought it up. Before I make a grand announcement that Lamson only deals with UTF-8 I better
know that it’s going to have minimal impact on people using it. I made a similar decision with
Mongrel, where I chose not to accept clients that didn’t follow the standard, and it turned
out to block a ton of security attacks for free.
What if Lamson’s required conversion to UTF-8 could not only simplify Lamson mail processing,
but also prevent spam and security flaws? I decided to do some basic analysis and find out.
This Is The Mailocalypse
I’ve decided that Lamson will NOT try to maintain the encodings it is given by a client.
It will consider all encodings suspect, and attempt to convert them to UTF-8 as a means
of cleaning and simplifying the mail it receives before it talks to other systems. Every
other system you’d most likely talk to is either UTF-8 capable or even requires it.
I’m calling this “The Mailocalypse” because it’s funny, but it will also separate Lamson from
other systems, in potentially awesome or horrible ways.
To validate this premise, I created a small Python script that takes a Maildir or mbox and tries to
convert every email into a UTF-8 encoding. It’s not trying to write the emails, just
load them and convert every single piece reliably to UTF-8. It then reports any emails
that have a problem with the conversion. After that I ran it on a bunch of mail I had
and got other people to run it and report their findings.
all is the total number of messages in the mailbox (including bad). bad is the number that were bad.
spam is whether that box was a spam box or not (meaning all messages were classified as spam).
I also made sure that the characters showed up in web browsers and in various UTF-8
mail clients and terminal windows. Most of the conversion was reliable and very
fast for a quick script.
With the above data, there’s two things you can notice right away, and which are confirmed with a quick
R session:
Spam fails to convert more frequently than ham messages. It looks like failure to convert is a 0.03 significant indicator that the message is spam. I’d need more data to confirm that.
Legit mail (ham) only has a 1.2% chance of failing to convert, and a quick eye-ball sample says those failures mail are mostly spam that wasn’t classified right.
Potential Problems
This analysis shows initially that converting mail to UTF-8 could work, but there are two problems
I see so far:
I may not be doing the best conversion in that script.
Encryption and signatures will screw this up.
For the first problem I’m posting the script and asking people to check it out and
let me know if there’s problems. Run it on your stuff and shoot me the ALL/BAD numbers
when it’s done. Check the code and see if you can improve it. Whatever you can to show
that converting to UTF-8 is a good OR bad idea.
A key design decision though will be that you never lose the original email, only that it
is converted for Lamson so you can work with it in your code without it blowing up. If the
mail can’t convert, then it’s bad email and should be rejected anyway. No point trying to
process it. For encrypted email or ones with signatures you would just run all the validation
on the original, or forward the original on, depending on your application.
For example, a mailing list would still want to convert all mail to UTF-8 for
processing, posting to web sites, spam filtering, and rejecting potentially bad
email. Signed and encrypted email would then just be forwarded on in original form,
or decoded and validated depending on the how the mailing list works.
Advice And Criticism Welcome
Take the mailocalypse script (presented below for review) and try it out. Confirm that
it works on most of your mail, and send me the results. Simplest way is to send me
a message on twitter like:
@zedshaw ALL 2340 BAD 17 spam yes #mailocalypse
And I’ll tally the results and do a better analysis.
You are also encouraged to fill me in on what I’m missing in the idea. Realize that part
of the idea is to eliminate uncommon corner cases and to prove that a situation is common
using evidence. Try to follow the same model by running the script on your mail to prove
that it won’t convert reliably or devise your own script to demonstrate your thoughts.
import email
from email.header import make_header, decode_header
from string import capwords
import sys
import mailbox
ALL_MAIL = 0
BAD_MAIL = 0
def all_parts(msg):
parts = [m for m in msg.walk() if m != msg]
if not parts:
parts = [msg]
return parts
def collapse_header(header):
if header.strip().startswith("=?"):
decoded = decode_header(header)
converted = (unicode(
x[0], encoding=x[1] or 'ascii', errors='replace')
for x in decoded)
value = u"".join(converted)
else:
value = unicode(header, errors='replace')
return value.encode("utf-8")
def convert_header_insanity(header):
if header is None:
return header
elif type(header) == list:
return [collapse_header(h) for h in header]
else:
return collapse_header(header)
def encode_header(name, val, charset='utf-8'):
msg[name] = make_header([(val, charset)]).encode()
def bless_headers(msg):
# go through every header and convert it to utf-8
headers = {}
for h in msg.keys():
headers[capwords(h, '-')] = convert_header_insanity(msg[h])
return headers
def dump_headers(headers):
for h in headers:
print h, headers[h]
def mail_load_cleanse(msg_file):
global ALL_MAIL
global BAD_MAIL
msg = email.message_from_file(msg_file)
headers = bless_headers(msg)
# go through every body and convert it to utf-8
parts = all_parts(msg)
bodies = []
for part in parts:
guts = part.get_payload(decode=True)
if part.get_content_maintype() == "text":
charset = part.get_charsets()[0]
try:
if charset:
uguts = unicode(guts, part.get_charsets()[0])
guts = uguts.encode("utf-8")
else:
guts = guts.encode("utf-8")
except UnicodeDecodeError, exc:
print >> sys.stderr, "CONFLICTED CHARSET:", exc, part.get_charsets()
BAD_MAIL += 1
except LookupError, exc:
print >> sys.stderr, "UNKNOWN CHARSET:", exc, part.get_charsets()
BAD_MAIL += 1
except Exception, exc:
print >> sys.stderr, "WEIRDO ERROR", exc, part.get_charsets()
BAD_MAIL += 1
ALL_MAIL += 1
mb = None
try:
mb = mailbox.Maildir(sys.argv[1])
len(mb) # need this to make the maildir try to read the directory and fail
except OSError:
print "NOT A MAILDIR, TRYING MBOX"
mb = mailbox.mbox(sys.argv[1])
if not mb:
print "NOT A MAILDIR OR MBOX, SORRY"
for key in mb.keys():
mail_load_cleanse(mb.get_file(key))
print >> sys.stderr, "ALL", ALL_MAIL
print >> sys.stderr, "BAD", BAD_MAIL
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-14.txt 0000644 0000765 0000024 00000021570 11215276254 023354 0 ustar zedshaw staff Title: The Mailocalypse Is Upon Us!
I'm currently polishing off the two final features before I start going for
the Lamson 1.0 release. I've been using Lamson to make a few little cute applications
and create one thing for a potential client, and so far I haven't had to change much
since 0.9.3. It's great so far and I hope that Lamson 1.0 will be a fun release.
The *big* thing that must improve though is handling character encodings in emails.
After spending a week or more trying to come up with an automated conversion scheme that
would honor all the encodings on the planet, I had a realization.
h2. Why Isn't All Mail UTF-8?
Remembering the purpose of Lamson as a *modern* mail server and framework, it finally
struck me as dumb that I'm trying to keep around encodings that were invented in the
pre-Unicode days. Every modern system understands and displays UTF-8, and apart from
some pissy attitudes from the Japanese about the Han unification, there's really no reason
that every email Lamson handles can't be "upgraded" to UTF-8.
As I thought about this more I started to like the idea. Take all mail Lamson handles, and
upgrade it to utf-8. Then inside Lamson you know that it's future proofed and ready to
be used by Python no matter what. Finally when the mail goes out, the conversion for sending
is simply to encode the email as proper UTF-8 encoding consistently.
However, I'm not the type of person to think any random idea I have is instantly valid because
I thought it up. Before I make a grand announcement that Lamson only deals with UTF-8 I better
know that it's going to have minimal impact on people using it. I made a similar decision with
Mongrel, where I chose *not* to accept clients that didn't follow the standard, and it turned
out to block a ton of security attacks for free.
What if Lamson's required conversion to UTF-8 could not only simplify Lamson mail processing,
but also prevent spam and security flaws? I decided to do some basic analysis and find out.
h2. This Is The Mailocalypse
I've decided that Lamson will *NOT* try to maintain the encodings it is given by a client.
It will consider all encodings suspect, and attempt to convert them to UTF-8 as a means
of cleaning and simplifying the mail it receives before it talks to other systems. Every
other system you'd most likely talk to is either UTF-8 capable or even requires it.
I'm calling this "The Mailocalypse" because it's funny, but it will also separate Lamson from
other systems, in potentially awesome or horrible ways.
To validate this premise, I created a small Python script that takes a Maildir or mbox and tries to
convert every email into a UTF-8 encoding. It's not trying to write the emails, just
load them and convert every single piece reliably to UTF-8. It then reports any emails
that have a problem with the conversion. After that I ran it on a bunch of mail I had
and got other people to run it and report their findings.
bq. "You can grab the mailocalypse.py script here.":/mailocalypse.py Improvements welcome.
The data I got from a small sample of the mail (randomly selected) was the following:
bq. all is the total number of messages in the mailbox (including bad). bad is the number that were bad.
spam is whether that box was a spam box or not (meaning all messages were classified as spam).
I also made sure that the characters showed up in web browsers and in various UTF-8
mail clients and terminal windows. Most of the conversion was reliable and very
fast for a quick script.
With the above data, there's two things you can notice right away, and which are confirmed with a quick
R session:
# Spam fails to convert more frequently than ham messages. It looks like failure to convert is a 0.03 significant indicator that the message is spam. I'd need more data to confirm that.
# Legit mail (ham) only has a 1.2% chance of failing to convert, and a quick eye-ball sample says those failures mail are mostly spam that wasn't classified right.
h2. Potential Problems
This analysis shows initially that converting mail to UTF-8 could work, but there are two problems
I see so far:
# I may not be doing the best conversion in that script.
# Encryption and signatures will screw this up.
For the first problem I'm posting the script and asking people to check it out and
let me know if there's problems. Run it on your stuff and shoot me the ALL/BAD numbers
when it's done. Check the code and see if you can improve it. Whatever you can to show
that converting to UTF-8 is a good *OR* bad idea.
A key design decision though will be that you never lose the original email, only that it
is converted for Lamson so you can work with it in your code without it blowing up. If the
mail can't convert, then it's bad email and should be rejected anyway. No point trying to
process it. For encrypted email or ones with signatures you would just run all the validation
on the original, or forward the original on, depending on your application.
For example, a mailing list would still want to convert all mail to UTF-8 for
processing, posting to web sites, spam filtering, and rejecting potentially bad
email. Signed and encrypted email would then just be forwarded on in original form,
or decoded and validated depending on the how the mailing list works.
h2. Advice And Criticism Welcome
Take the mailocalypse script (presented below for review) and try it out. Confirm that
it works on most of your mail, and send me the results. Simplest way is to send me
a message on twitter like:
bq. @zedshaw ALL 2340 BAD 17 spam yes #mailocalypse
And I'll tally the results and do a better analysis.
You are also encouraged to fill me in on what I'm missing in the idea. Realize that *part*
of the idea is to eliminate uncommon corner cases and to prove that a situation is common
using evidence. Try to follow the same model by running the script on your mail to prove
that it won't convert reliably or devise your own script to demonstrate your thoughts.
h2. The Codes
"You can grab the mailocalypse.py script here.":/mailocalypse.py Improvements welcome.
import email
from email.header import make_header, decode_header
from string import capwords
import sys
import mailbox
ALL_MAIL = 0
BAD_MAIL = 0
def all_parts(msg):
parts = [m for m in msg.walk() if m != msg]
if not parts:
parts = [msg]
return parts
def collapse_header(header):
if header.strip().startswith("=?"):
decoded = decode_header(header)
converted = (unicode(
x[0], encoding=x[1] or 'ascii', errors='replace')
for x in decoded)
value = u"".join(converted)
else:
value = unicode(header, errors='replace')
return value.encode("utf-8")
def convert_header_insanity(header):
if header is None:
return header
elif type(header) == list:
return [collapse_header(h) for h in header]
else:
return collapse_header(header)
def encode_header(name, val, charset='utf-8'):
msg[name] = make_header([(val, charset)]).encode()
def bless_headers(msg):
# go through every header and convert it to utf-8
headers = {}
for h in msg.keys():
headers[capwords(h, '-')] = convert_header_insanity(msg[h])
return headers
def dump_headers(headers):
for h in headers:
print h, headers[h]
def mail_load_cleanse(msg_file):
global ALL_MAIL
global BAD_MAIL
msg = email.message_from_file(msg_file)
headers = bless_headers(msg)
# go through every body and convert it to utf-8
parts = all_parts(msg)
bodies = []
for part in parts:
guts = part.get_payload(decode=True)
if part.get_content_maintype() == "text":
charset = part.get_charsets()[0]
try:
if charset:
uguts = unicode(guts, part.get_charsets()[0])
guts = uguts.encode("utf-8")
else:
guts = guts.encode("utf-8")
except UnicodeDecodeError, exc:
print >> sys.stderr, "CONFLICTED CHARSET:", exc, part.get_charsets()
BAD_MAIL += 1
except LookupError, exc:
print >> sys.stderr, "UNKNOWN CHARSET:", exc, part.get_charsets()
BAD_MAIL += 1
except Exception, exc:
print >> sys.stderr, "WEIRDO ERROR", exc, part.get_charsets()
BAD_MAIL += 1
ALL_MAIL += 1
mb = None
try:
mb = mailbox.Maildir(sys.argv[1])
len(mb) # need this to make the maildir try to read the directory and fail
except OSError:
print "NOT A MAILDIR, TRYING MBOX"
mb = mailbox.mbox(sys.argv[1])
if not mb:
print "NOT A MAILDIR OR MBOX, SORRY"
for key in mb.keys():
mail_load_cleanse(mb.get_file(key))
print >> sys.stderr, "ALL", ALL_MAIL
print >> sys.stderr, "BAD", BAD_MAIL
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-20.html 0000644 0000765 0000024 00000021233 11230770144 023464 0 ustar zedshaw staff
LamsonProject: Lamson 0.9.4 With Unicode Super Powers
Lamson 0.9.4 is out and it’s sporting a completely rewritten and meticulously
crafted encoding system. With the new lamson.encoding code Lamson can now decode nearly any nasty horrible encoded spam
or mail you hand it, turn it into pristine nice Python unicode strings, and
then output sweet clean ascii or utf-8 in a consistent way.
The purpose of this new encoding system is to make sure that Lamson
is giving your handlers the best input it can, based on the assumption
that the world is evil and Lamson will be handed utter garbage.
In order to pull this off, Lamson actually attempts to decode everything
it gets, and uses chardet to guess whenever
it can’t. It goes far enough to make sure that 99.9% of legit mail gets through,
and the rest is usually spam. However, Lamson is so good at this conversion
now that it even does it properly on about 99% of spam, even more.
It doesn’t matter what character encoding is used, if there’s a mix of encodings
or even if something lies about the encoding or doesn’t give one. Lamson can
figure this out and convert it anyway, and the rest is junk.
You can look at this image of a spam inbox and
this image of a spam inbox where you can see a
screen session showing this off. This is a split screen with a Mutt on top
showing badly formatted spam, and then a Mutt on the bottom with the
results of Lamson’s cleansing. This is just running in iTerm, but looks
the same in most unicode enabled terminals and clients.
The interesting side effect of Lamson’s decoding is that it undoes almost all
of the spam obfuscation techniques quickly and consistently.
There Will Be Bugs
I’ve thrashed the hell out of this code and made sure that I really took
the time to get it right. I’ve ran it on thousands and thousands of real and
spam mail and tested the crap out of it with unit tests and fuzzing.
Still, this is new code handling email so there will be bugs in it. If
you run into an email that you think should parse correctly, send it to
me and I’ll look at it.
How It Works
All Lamson does is completely parse every header and figure out how to decode it,
but it assumes that the client is a liar. If the string decodes without errors
then Lamson is happy, but if it blows up then Lamson uses chardet to figure out
what the contents really are encoded as. If chardet can’t figure it out, or if
it’s still busted, then Lamson rejects that mail (which happens less than a fraction
of a percent of the time to real email).
Once this decoding process is finished Lamson has converted the email completely
into Python unicode only. You can work with it in a modern way without worrying
about the original codecs, and you can set your headers and attachments without
worrying about setting the codecs because Lamson will use the same tech to get
the encoding right.
When Lamson writes an email, it assumes that your text can be encoded as either
plain ASCII (as most headers are), or UTF-8. If it can’t, then your text
probably can’t be processed by Lamson anyway. Lamson will favor ASCII first since
it’s easier, and then use UTF-8 for anything that can’t be ASCII.
During this conversion process anything that’s not text is treated as raw binary
and just decoded as-is.
By doing this Lamson produces nice clean email that can be easily processed and
passed around, and which you can review and debug easier. It also turns Lamson
into a “modernizing” agent since it is producing the email it wants to see.
Getting This Release
I had some issues pushing this release to PyPI, so let me know if it barfs on you.
Otherwise, you can hit the download section and grab the
gear.
The New “Cleanse” Command
In order to make it easier for people to try this new cleansing system on real
email I’ve written a quick lamson cleanse command. What this command will do is
take a mbox or Maildir, and run it through the washing machine, writing the results
to a different maildir. When it’s done you can go look at the reasons why some
mail failed, how many failures, and you can actually open that maildir mail and
look at it with your client.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-20.txt 0000644 0000765 0000024 00000010450 11217126716 023343 0 ustar zedshaw staff Title: Lamson 0.9.4 With Unicode Super Powers
Lamson 0.9.4 is out and it's sporting a completely rewritten and meticulously
crafted encoding system. With the new "lamson.encoding":http://lamsonproject.org/docs/api/lamson.encoding-module.html code Lamson can now decode nearly any nasty horrible encoded spam
or mail you hand it, turn it into pristine nice Python unicode strings, and
then output sweet clean ascii or utf-8 in a consistent way.
The purpose of this new encoding system is to make sure that Lamson
is giving your handlers the best input it can, based on the assumption
that the world is evil and Lamson will be handed utter garbage.
In order to pull this off, Lamson actually attempts to decode everything
it gets, and uses "chardet":http://chardet.feedparser.org/ to guess whenever
it can't. It goes far enough to make sure that 99.9% of legit mail gets through,
and the rest is usually spam. However, Lamson is so good at this conversion
now that it even does it properly on about 99% of spam, even more.
It doesn't matter what character encoding is used, if there's a mix of encodings
or even if something lies about the encoding or doesn't give one. Lamson can
figure this out and convert it anyway, and the rest is junk.
You can look at this "image of a spam inbox":/lamson_vs_spam.png and
"this image of a spam inbox":/lamson_vs_spam_2.png where you can see a
screen session showing this off. This is a split screen with a Mutt on top
showing badly formatted spam, and then a Mutt on the bottom with the
results of Lamson's cleansing. This is just running in iTerm, but looks
the same in most unicode enabled terminals and clients.
The interesting side effect of Lamson's decoding is that it undoes almost all
of the spam obfuscation techniques quickly and consistently.
h2. There Will Be Bugs
I've thrashed the hell out of this code and made sure that I really took
the time to get it right. I've ran it on thousands and thousands of real and
spam mail and tested the crap out of it with unit tests and fuzzing.
Still, this is new code handling email so there will be bugs in it. If
you run into an email that you think should parse correctly, send it to
me and I'll look at it.
h2. How It Works
All Lamson does is completely parse every header and figure out how to decode it,
but it assumes that the client is a liar. If the string decodes without errors
then Lamson is happy, but if it blows up then Lamson uses chardet to figure out
what the contents really are encoded as. If chardet can't figure it out, or if
it's still busted, then Lamson rejects that mail (which happens less than a fraction
of a percent of the time to real email).
Once this decoding process is finished Lamson has converted the email completely
into Python unicode only. You can work with it in a modern way without worrying
about the original codecs, and you can set your headers and attachments without
worrying about setting the codecs because Lamson will use the same tech to get
the encoding right.
When Lamson writes an email, it assumes that your text can be encoded as either
plain ASCII (as most headers are), or UTF-8. If it can't, then your text
probably can't be processed by Lamson anyway. Lamson will favor ASCII first since
it's easier, and then use UTF-8 for anything that can't be ASCII.
bq. During this conversion process anything that's *not* text is treated as raw binary
and just decoded as-is.
By doing this Lamson produces nice clean email that can be easily processed and
passed around, and which you can review and debug easier. It also turns Lamson
into a "modernizing" agent since it is producing the email it wants to see.
h2. Getting This Release
I had some issues pushing this release to PyPI, so let me know if it barfs on you.
Otherwise, you can hit the "download":/download.html section and grab the
gear.
h2. The New "Cleanse" Command
In order to make it easier for people to try this new cleansing system on real
email I've written a quick @lamson cleanse@ command. What this command will do is
take a mbox or Maildir, and run it through the washing machine, writing the results
to a different maildir. When it's done you can go look at the reasons why some
mail failed, how many failures, and you can actually open that maildir mail and
look at it with your client.
Try it out and report any errors you find.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-22.html 0000644 0000765 0000024 00000020405 11230770144 023466 0 ustar zedshaw staff
LamsonProject: 0.9.5 Almost There, But Stumped On Templates
Since the 0.9.4 release I’ve rewritten the main part of the decoding parser
so that it’s much cleaner and handles more edge conditions. If there’s one
word that defines what makes MIME horrible it would be “edge”.
It’s amazing the kind of stupid crap clients are allowed to send out in headers.
I’ve actually got test cases from Mutt where it breaks headers up into completely
different encodings across multiple lines for no damn reason at all. Here’s a
test I had to write just to cover the one worst case I found:
def test_dumb_shit():
# this is a sample of possibly the worst case Mutt can produce
idiot = '=?iso-8859-1?B?SOlhdnkgTel05WwgVW7uY/hk?=\n\t=?iso-8859-1?Q?=E9?='
should_be = u'H\xe9avy M\xe9t\xe5l Un\xeec\xf8d\xe9'
assert_equal(encoding.header_from_mime_encoding(idiot), should_be)
If you can decode what that actually is and why it’s retarded then you’ll win a
prize. I’ll give you a hint: B != Q.
I originally wanted to make writing templates as easy as writing an email or a web page,
so I let you do things like this:
To: {{ message['from'] }}
From: {{ confirm_address }}@{{host}}
Subject: Please confirm your new blog {{ post_name }}
Reply-To: {{ confirm_address }}@{{host}}
Hi there, you requested that I create a blog with a post named:
{{ post_name }} : {{ message['subject'] }}
...
This is a Jinja2 that oneshotblog.com sends
out when it makes you confirm creating a blog.
This seems all normal, since it’s the headers you want and the body,
like an email, but it is a total fantasy. The reason is you can’t actually
decode the above template if any part of it has non-ascii chars. I’ll explain, and
then hopefully I can get some suggestions on what people would like.
First, Lamson does the right thing and in your templates you are getting perfectly
usable and future proof unicode objects to work with in your handlers. Also,
Jinja2 has no problem with taking this unicode and processing the above template.
It works fine, and Jinja2 or Mako aren’t to blame for the problem with
templates now.
The problem is actually in how email has to be parsed, since the email library
expects you to hand it a ascii encoded string, no unicode allowed. Nothing else.
Lamson’s new code can handle the unicode fine, the problem is actually in
this one function:
The above function has to go, but where I’m stuck is that it’d be way more
elegant to ditch this whole mess completely and remove headers from
the template.
It’s too hard to write these headers by hand unless they are simple ones,
and there needs to be a way to pass in more complex headers that you want
to add. If I’m providing a way to pass in headers to lamson.view.respond
then why not take the headers out of the template and simplify the
whole rendering process?
I’d like to hear from people using Lamson or those who might have ideas on this.
If taking your headers out of the template and moving them to the render call
will totally screw up your day, then let me know. If it comes down to it, and I
have to do it, I’ll probably write a conversion tool to help.
Also, if you have a suggestion for doing the above better but keeping the headers
then let me know.
Thanks for your help folks, and I’ll be hanging out in the #lamson channel on irc.freenode.org
if people want to hash out a design.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-22.txt 0000644 0000765 0000024 00000007765 11220041004 023340 0 ustar zedshaw staff Title: 0.9.5 Almost There, But Stumped On Templates
Since the 0.9.4 release I've rewritten the main part of the decoding parser
so that it's much cleaner and handles more edge conditions. If there's one
word that defines what makes MIME horrible it would be "edge".
It's amazing the kind of stupid crap clients are allowed to send out in headers.
I've actually got test cases from Mutt where it breaks headers up into completely
different encodings across multiple lines for no damn reason at all. Here's a
test I had to write just to cover the one worst case I found:
def test_dumb_shit():
# this is a sample of possibly the worst case Mutt can produce
idiot = '=?iso-8859-1?B?SOlhdnkgTel05WwgVW7uY/hk?=\n\t=?iso-8859-1?Q?=E9?='
should_be = u'H\xe9avy M\xe9t\xe5l Un\xeec\xf8d\xe9'
assert_equal(encoding.header_from_mime_encoding(idiot), should_be)
If you can decode what that actually is and why it's retarded then you'll win a
prize. I'll give you a hint: @B != Q@.
The example of the above working is "on this page at oneshotblog.com":http://oneshotblog.com/posts/zedshaw@zedshaw.com/test.html so that's solved (for now).
h2. Now About Those Templates
I originally wanted to make writing templates as easy as writing an email or a web page,
so I let you do things like this:
To: {{ message['from'] }}
From: {{ confirm_address }}@{{host}}
Subject: Please confirm your new blog {{ post_name }}
Reply-To: {{ confirm_address }}@{{host}}
Hi there, you requested that I create a blog with a post named:
{{ post_name }} : {{ message['subject'] }}
...
This is a "Jinja2":http://jinja.pocoo.org/2/ that "oneshotblog.com":oneshotblog.com sends
out when it makes you confirm creating a blog.
This seems all normal, since it's the headers you want and the body,
like an email, but it is *a total fantasy*. The reason is you can't actually
decode the above template if any part of it has non-ascii chars. I'll explain, and
then hopefully I can get some suggestions on what people would like.
First, Lamson does the right thing and in your templates you are getting perfectly
usable and future proof @unicode@ objects to work with in your handlers. Also,
Jinja2 has no problem with taking this unicode and processing the above template.
It works fine, and Jinja2 or Mako aren't to blame for the problem with
templates now.
The problem is actually in how email has to be parsed, since the email library
expects you to hand it a @ascii@ encoded string, no unicode allowed. Nothing else.
Lamson's new code can handle the unicode fine, the problem is actually in
this one function:
The above function has to go, but where I'm stuck is that it'd be way more
elegant to ditch this whole mess completely and remove headers from
the template.
It's too hard to write these headers by hand unless they are simple ones,
and there needs to be a way to pass in more complex headers that you want
to add. If I'm providing a way to pass in headers to @lamson.view.respond@
then why not take the headers out of the template and simplify the
whole rendering process?
I'd like to hear from people using Lamson or those who might have ideas on this.
If taking your headers out of the template and moving them to the render call
will totally screw up your day, then let me know. If it comes down to it, and I
have to do it, I'll probably write a conversion tool to help.
Also, if you have a suggestion for doing the above better but keeping the headers
then let me know.
Thanks for your help folks, and I'll be hanging out in the #lamson channel on irc.freenode.org
if people want to hash out a design.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-26.html 0000644 0000765 0000024 00000020236 11230770144 023474 0 ustar zedshaw staff
LamsonProject: Lamson 0.9.5, The Push To 1.0
I just released Lamson 0.9.5 with all the major Unicode refactoring done and working.
This is an important release because 0.9.5 is where I declare that I’m pushing to a 1.0
release and the base Lamson APIs won’t change under penalty of death. In fact they can’t
change because I’m using them myself in a few applications.
From now on, if an API has to change, then I’ll keep the old one around and attach a
deprecation warning, and then provide a new one. You’ll then have by 1.0 to stop using
it and start using the new one. Except for a situations where a bug must be fixed, I
can’t see the actual Lamson API needing such a drastic change by 1.0.
What will happen between now and 1.0 is bug fixing, and adding any nice additional
features people need within the current API to help build applications.
This would be things like more connectors to other mail protocols (LMTP, QPMP), commands
for deployment features, and possibly a way to write your own command.
So, if you’ve been waiting for the API to stabilize then 0.9.5 is the release to grab
and start using. I’m currently using it on about 6 little projects and it’s working
great so far, but I need more people to try it out and send me feedback on how well
it works for them.
Getting This Release
As usual, you can read the download documentation to learn how to
grab it.
The oneshotblog.com sample application is now deployed in a
new virtualhost configuration on a new VPS I purchased this week. This configuration
is different from the ones given in the documentation above because it allows you
to run mulitple applications for multiple domains on a single machine.
I’ll be writing up how to build your own version of this configuration today, but the
quick description is this:
I have postfix running on port 25 now in front of Lamson.
Postfix is configured to take all mail for each domain and dump it into a maildir queue for each one.
The Lamson applications are then reconfigured in their config/boot.py to use a QueueReceiver to feed itself messages from its maildir.
Lamson then just uses the same postfix server as its relay host like before.
The advantage of this configuration, apart from being able to easily host multiple domains,
is that you can shutdown your Lamson application and when you start it back up it
will catch up on the mail in the maildir. People will probably never know you did
maintenance or stopped the server temporarily.
The disadvantage of this configuration is that it’s a royal pain to make postfix do this.
The instructions will show you step-by-step how to pull it off, so hopefully it’ll make
it easier on people who need it. However, I’m gonna tell people to start small and
just use the simpler configuration “Lamson in front” configuration and leave this
for the big times.
Mailing List Will Go Down
Finally, I’ll be moving the mailing list shortly, so if you send a message and don’t receive
a reply then that’s why.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-06-26.txt 0000644 0000765 0000024 00000007241 11221170627 023350 0 ustar zedshaw staff Title: Lamson 0.9.5, The Push To 1.0
I just released Lamson 0.9.5 with all the major Unicode refactoring done and working.
This is an important release because 0.9.5 is where I declare that I'm pushing to a 1.0
release and the base Lamson "APIs":/docs/api/ won't change under penalty of death. In fact they can't
change because I'm using them myself in a few applications.
From now on, if an API has to change, then I'll keep the old one around and attach a
deprecation warning, and then provide a new one. You'll then have by 1.0 to stop using
it and start using the new one. Except for a situations where a bug must be fixed, I
can't see the actual Lamson API needing such a drastic change by 1.0.
What will happen between now and 1.0 is bug fixing, and adding any nice additional
features people need within the current "API":/docs/api/ to help build applications.
This would be things like more connectors to other mail protocols (LMTP, QPMP), commands
for deployment features, and possibly a way to write your own command.
So, if you've been waiting for the API to stabilize then 0.9.5 is the release to grab
and start using. I'm currently using it on about 6 little projects and it's working
great so far, but I need more people to try it out and send me feedback on how well
it works for them.
h2. Getting This Release
As usual, you can read the "download":/download.html documentation to learn how to
grab it.
After you grab it, go read these documents:
* "Getting Started With Lamson":http://lamsonproject.org/docs/getting_started.html
* "A Painless Introduction To Finite State Machines":http://lamsonproject.org/docs/introduction_to_finite_state_machines.html
* "Deploying Lamson And OneShotBlog":http://lamsonproject.org/docs/deploying_lamson.html
Then you should hit the "API documentation":http://lamsonproject.org/docs/api/ and
start writing your application. Keep your first one small and try deploying it
in different ways.
h2. Simple Virtualhost Deployment Instructions Coming Soon
The "oneshotblog.com":http://oneshotblog.com/ sample application is now deployed in a
new virtualhost configuration on a new VPS I purchased this week. This configuration
is different from the ones given in the documentation above because it allows you
to run mulitple applications for multiple domains on a single machine.
I'll be writing up how to build your own version of this configuration today, but the
quick description is this:
# I have postfix running on port 25 now *in front* of Lamson.
# Postfix is configured to take all mail for each domain and dump it into a maildir queue for each one.
# The Lamson applications are then reconfigured in their @config/boot.py@ to use a "QueueReceiver":http://lamsonproject.org/docs/api/lamson.server.QueueReceiver-class.html to feed itself messages from its maildir.
# Lamson then just uses the same postfix server as its relay host like before.
The advantage of this configuration, apart from being able to easily host multiple domains,
is that you can shutdown your Lamson application and when you start it back up it
will catch up on the mail in the maildir. People will probably never know you did
maintenance or stopped the server temporarily.
The disadvantage of this configuration is that it's a royal pain to make postfix do this.
The instructions will show you step-by-step how to pull it off, so hopefully it'll make
it easier on people who need it. However, I'm gonna tell people to start small and
just use the simpler configuration "Lamson in front" configuration and leave this
for the big times.
h2. Mailing List Will Go Down
Finally, I'll be moving the mailing list shortly, so if you send a message and don't receive
a reply then that's why.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-07-03.html 0000644 0000765 0000024 00000012363 11230770144 023472 0 ustar zedshaw staff
LamsonProject: Article In The Reg About Lamson (By Ted Dziuba)
A snippet from the article: “As a development platform, e-mail has gone
neglected for decades. Its esoteric implementation details and specifications
are regarded by many in the IT business as voodoo, best left to old-granddad
programs like Sendmail or Postfix. Zed Shaw hopes to change that with his new
project, Lamson. (The name Lamson is a throwback to the early 20th century
pneumatic tubes used to shuttle messages between offices. It was
originally called Son of Sam, but Zed’s dog convinced him to change
it).”
I’ve been cranking on various projects with Lamson and haven’t ran into too many other bugs. I’ll be
doing a code review of what’s in the Bazaar repo and then pusing out a 0.9.6 probably this weekend.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-07-03.txt 0000644 0000765 0000024 00000002330 11223705276 023344 0 ustar zedshaw staff Title: Article In The Reg About Lamson (By Ted Dziuba)
Just a quick update to point people at an article in "The Register":http://www.theregister.co.uk/ by
"Ted Dziuba":http://teddziuba.com/ entitled "Lamson - email app coding without the palm sweat: Doing what Java never did":http://www.theregister.co.uk/2009/07/03/lamson/. He interviewed me about Lamson, things I think you can use it for, and other fun stuffs.
A snippet from the article: "As a development platform, e-mail has gone
neglected for decades. Its esoteric implementation details and specifications
are regarded by many in the IT business as voodoo, best left to old-granddad
programs like Sendmail or Postfix. Zed Shaw hopes to change that with his new
project, Lamson. (The name Lamson is a throwback to the early 20th century
pneumatic tubes used to shuttle messages between offices. It was
originally called Son of Sam, but Zed's dog convinced him to change
it)."
"Read the article.":http://www.theregister.co.uk/2009/07/03/lamson/
h2. 0.9.6 Coming Soon
I've been cranking on various projects with Lamson and haven't ran into too many other bugs. I'll be
doing a code review of what's in the Bazaar repo and then pusing out a 0.9.6 probably this weekend.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-07-07.html 0000644 0000765 0000024 00000010160 11230770144 023467 0 ustar zedshaw staff
LamsonProject: Lamson Mailing Lists Down
I’m making up the release of Lamson and doing some server maintenance so I took down the
Lamson server running the mailing lists. I’ll let everyone know when they’re back up.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-07-07.txt 0000644 0000765 0000024 00000000324 11224721637 023351 0 ustar zedshaw staff Title: Lamson Mailing Lists Down
I'm making up the release of Lamson and doing some server maintenance so I took down the
Lamson server running the mailing lists. I'll let everyone know when they're back up.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-07-09.html 0000644 0000765 0000024 00000033154 11230770144 023501 0 ustar zedshaw staff
LamsonProject: Lamson's Bounce Detection Algorithm
I just finished committing 0.9.6 code for doing bounce detection and analysis
with Lamson. It’s part of the new mailing list example I’m coding up for the
0.9.6 release which I’ll be running on a free mailing list site I’m going
to release soon. In this blog post I’d like to go through the bounce detection
algorithm and get some feedback and samples from people. So far it works great for
the samples I have, but I want it to be fairly bullet proof.
What I have here is a quick description of how Lamson is doing bounce detection,
and then how it gives you parsed information to do analysis beyond that. I’m hoping
that people who have to deal with bounces from systems like MS Exchange can possibly
either provide samples or advice so I can tune the algorithm better.
In RFC3464 there’s various headers that are given which a bounce message should have,
as well as the double and sometimes triple multipart-mime wrapping you need to do
to give back the bounced message. A major pain with this standard is that the headers
are spread out across the different nested parts, some might be duplicated, and the format
is never respected by the various MTAs out there.
With RFC1893 you have a large number of status codes, but they are combined in weird ways
such that you have a primary code with the first digit, a secondary code with the
second, and then a “third” code that’s the second and third digit combined. Unlike HTTP
error codes where there’s just a single number mapped to a single status code, in RFC1893
it’s this strangely nested first+second+(second+third) thing.
Adding to this is there’s a few headers where you can put these status codes, and in
various formats. Here’s a sample of the parsed Diagnostic-Code from a gmail bounce
message:
'Diagnostic-Code': [(u'smtp',
u'550-5.1.1',
u"The email account that you tried to reach does...")],
The end result is that if you want to determine if a message has bounce (not even hard vs. soft)
then you have to go trolling through nested attachments looking for headers and parsing their values
on the chance they might be important to the bounce.
Soft and Hard
The general consensus is that if a primary status code is greater than 4 then
the message is a hard bounce, anything else is a soft bounce. Of course there’s no way
this is an absolute truth, and I’m sure there will end up being various complex
things people will want to do with a soft vs. a hard bounce.
Since there’s no way Lamson could know what you want to base your decisions to honor a
soft vs. hard bounce, it simply lets you figure out which type it is, and then analyze
the information it’s collected to determine the bounce.
Lamson’s Simplification
Lamson already converts the email into a normalized form, so the difficult part of
parsing out the bounce information is simply trolling through the parts and finding
all these randomly dispersed bounce headers. Then it’s a matter of parsing the values
into something that can be use for analysis in code.
To simplify this, Lamson uses a dict of headers and matching regexes to search for in
the email:
Each of these headers shows up about once, maybe more in the parts of a message
that has bounced information, and they don’t show up at all or rarely in regular
messages. What Lamson does is just walk through each part, finds the matches, parses
with the regex, and then produces something like this:
I’ve abbreviated some of the messages since they get stupidly long, but this shows you the information
you get now. Lamson also provides a nice API through the lamson.bounce.BounceAnalyzer class that
you’ll see in a moment.
If you know of additional headers and formats that show up in bounces, let me know.
Bounce Probability
Now, we’ve collected up all the different headers you might find in a bounce package, and we’ve
attempted to parse the values into something meaningful. That means, the more of these headers
that we find and successfully parse, the more likely the message is a bounce.
Lamson keeps a score while it’s finding the above headers, and for each one it finds and parses
it adds a “point”. It then produces a probability <= 1.0 that the message is a bounce or not.
The more headers it finds, the closer to 1.0, the higher the chance. So far it looks like any
message above 0.3 is most likely a bounce message, with only a few spams that are below that.
Which is from the test suite, and shows you various ways to get information
about the probability of the message being a bounce.
Bounce Header Information
Finally, you’ll need to get at this parsed information to make any
final decisions about the bounce. You might want to keep track of
which MTAs consistently fail and include them in your black lists.
You might want to tune your retries for different kind of soft
bounces.
Whatever you need to do, you have the original raw headers that
Lamson found, but you also have a higher level API you can use:
This demonstrates how I’ve pulled out all of the error codes mentioned in
RFC1893 and put them into Lamson so that you can get meaningful error
messages for the various multi-level error codes you’ll encounter.
All of these error codes are available in the lamson.bounce module.
Comments Or Suggestions
That’s it for the review of Lamson’s bounce detection. It’s in the early
stages but already showing a lot of promise. It turned out to be much
easier than I anticipated, but only after doing the work in lamson.encoding
to clean up emails.
If you can think of situations that should be covered, and if you have samples
of horrible bounce messages, then I’d love to have them.
Mailing Lists Coming Back
Tomorrow I’ll be using this to reimplement the mailing list sample and put
it back into the bzr repository, then back online. I’ll have a nice
status update about that tomorrow as I finish it up.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-07-09.txt 0000644 0000765 0000024 00000022024 11225570307 023351 0 ustar zedshaw staff Title: Lamson's Bounce Detection Algorithm
I just finished committing 0.9.6 code for doing bounce detection and analysis
with Lamson. It's part of the new mailing list example I'm coding up for the
0.9.6 release which I'll be running on a free mailing list site I'm going
to release soon. In this blog post I'd like to go through the bounce detection
algorithm and get some feedback and samples from people. So far it works great for
the samples I have, but I want it to be fairly bullet proof.
What I have here is a quick description of how Lamson is doing bounce detection,
and then how it gives you parsed information to do analysis beyond that. I'm hoping
that people who have to deal with bounces from systems like MS Exchange can possibly
either provide samples or advice so I can tune the algorithm better.
h2. The State of RFC 3464 And 1893
The two big RFCs for bounce handling are "RFC3464 - An Extensible Message Format for Delivery Status Notifications":http://www.faqs.org/rfcs/rfc3464.html
and "RFC1893 - Enhanced Mail System Status Codes":http://www.faqs.org/rfcs/rfc1893.html which
cover the headers and error messages you'll encounter.
In RFC3464 there's various headers that are given which a bounce message should have,
as well as the double and sometimes triple multipart-mime wrapping you need to do
to give back the bounced message. A major pain with this standard is that the headers
are spread out across the different nested parts, some might be duplicated, and the format
is *never* respected by the various MTAs out there.
With RFC1893 you have a large number of status codes, but they are combined in weird ways
such that you have a primary code with the first digit, a secondary code with the
second, and then a "third" code that's the second and third digit combined. Unlike HTTP
error codes where there's just a single number mapped to a single status code, in RFC1893
it's this strangely nested first+second+(second+third) thing.
Adding to this is there's a few headers where you can put these status codes, and in
various formats. Here's a sample of the *parsed* Diagnostic-Code from a gmail bounce
message:
'Diagnostic-Code': [(u'smtp',
u'550-5.1.1',
u"The email account that you tried to reach does...")],
The end result is that if you want to determine if a message has bounce (not even hard vs. soft)
then you have to go trolling through nested attachments looking for headers and parsing their values
on the chance they *might* be important to the bounce.
h2. Soft and Hard
The general consensus is that if a *primary status code* is greater than 4 then
the message is a *hard* bounce, anything else is a soft bounce. Of course there's no way
this is an absolute truth, and I'm sure there will end up being various complex
things people will want to do with a soft vs. a hard bounce.
Since there's no way Lamson could know what you want to base your decisions to honor a
soft vs. hard bounce, it simply lets you figure out which type it is, and then analyze
the information it's collected to determine the bounce.
h2. Lamson's Simplification
Lamson already converts the email into a normalized form, so the difficult part of
parsing out the bounce information is simply trolling through the parts and finding
all these randomly dispersed bounce headers. Then it's a matter of parsing the values
into something that can be use for analysis in code.
To simplify this, Lamson uses a dict of headers and matching regexes to search for in
the email:
Each of these headers shows up about once, maybe more in the parts of a message
that has bounced information, and they don't show up at all or rarely in regular
messages. What Lamson does is just walk through each part, finds the matches, parses
with the regex, and then produces something like this:
I've abbreviated some of the messages since they get stupidly long, but this shows you the information
you get now. Lamson also provides a nice API through the lamson.bounce.BounceAnalyzer class that
you'll see in a moment.
bq. If you know of additional headers and formats that show up in bounces, let me know.
h2. Bounce Probability
Now, we've collected up all the different headers you might find in a bounce package, and we've
attempted to parse the values into something meaningful. That means, the more of these headers
that we find and successfully parse, the more likely the message is a bounce.
Lamson keeps a score while it's finding the above headers, and for each one it finds and parses
it adds a "point". It then produces a probability <= 1.0 that the message is a bounce or not.
The more headers it finds, the closer to 1.0, the higher the chance. So far it looks like any
message above 0.3 is most likely a bounce message, with only a few spams that are below that.
Checking for a bounce then looks like this:
Which is from the test suite, and shows you various ways to get information
about the probability of the message being a bounce.
h2. Bounce Header Information
Finally, you'll need to get at this parsed information to make any
final decisions about the bounce. You might want to keep track of
which MTAs consistently fail and include them in your black lists.
You might want to tune your retries for different kind of soft
bounces.
Whatever you need to do, you have the original raw headers that
Lamson found, but you also have a higher level API you can use:
This demonstrates how I've pulled out all of the error codes mentioned in
RFC1893 and put them into Lamson so that you can get meaningful error
messages for the various multi-level error codes you'll encounter.
All of these error codes are available in the lamson.bounce module.
h2. Comments Or Suggestions
That's it for the review of Lamson's bounce detection. It's in the early
stages but already showing a lot of promise. It turned out to be much
easier than I anticipated, but only after doing the work in lamson.encoding
to clean up emails.
If you can think of situations that should be covered, and if you have samples
of horrible bounce messages, then I'd love to have them.
h2. Mailing Lists Coming Back
Tomorrow I'll be using this to reimplement the mailing list sample and put
it back into the bzr repository, then back online. I'll have a nice
status update about that tomorrow as I finish it up.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-07-14.html 0000644 0000765 0000024 00000011775 11230770144 023502 0 ustar zedshaw staff
LamsonProject: Lamson 0.9.6 Sometime Today
I think I’m at a point where bounce detection and handling works really well and is
ready for release. I’ve got it running on the latest Lamson demo librelist.com
and it works. I’ll be pushing out a new release of Libre List in a little while
that does some advanced bounce handling, and if that all works, then I’ll push
out 0.9.6.
This release will include bounce handling, the ability to change how state is
stored and maintained, the ability to route on the envelope rather than the
headers, and the full code to the librelist.com site including Django Integration
which really isn’t “integration”.
The plans for the 0.9.7 release are to remove all dependencies on SQLAlchemy to make
it clear that you don’t have to use it, and any bug fixes. After that release, assuming
there’s nothing major I need to add, I’ll be doing the 1.0 pre-releases.
Shoot me email if you have questions, or hop on #lamson on irc.freenode.org to chat.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-07-14.txt 0000644 0000765 0000024 00000002006 11227105220 023331 0 ustar zedshaw staff Title: Lamson 0.9.6 Sometime Today
I think I'm at a point where bounce detection and handling works really well and is
ready for release. I've got it running on the latest Lamson demo "librelist.com":http://librelist.com/
and it works. I'll be pushing out a new release of Libre List in a little while
that does some advanced bounce handling, and if that all works, then I'll push
out 0.9.6.
This release will include bounce handling, the ability to change how state is
stored and maintained, the ability to route on the envelope rather than the
headers, and the full code to the librelist.com site including "Django Integration":http://dpaste.de/3qGB/
which really isn't "integration".
The plans for the 0.9.7 release are to remove all dependencies on SQLAlchemy to make
it clear that you don't have to use it, and any bug fixes. After that release, assuming
there's nothing major I need to add, I'll be doing the 1.0 pre-releases.
Shoot me email if you have questions, or hop on #lamson on irc.freenode.org to chat.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-07-19.html 0000644 0000765 0000024 00000014752 11230770144 023505 0 ustar zedshaw staff
LamsonProject: I Blame Bounces For The Delay
I was going to release Lamson 0.9.6 much earlier than this, but as I worked on
the bounce detection feature, I realized that if I just wrote that and a few
other cleanups and features, I’d actually have a 1.0pre1 candidate on my
hands instead of a 0.9.6 release.
Instead of releasing 0.9.6, I continued to work on librelist.com
and got the software smart as hell. While working on librelist.com I managed
to improve Lamson to the point where I’ll be comfortable releasing a 1.0pre1
tomorrow.
What’s librelist.com?
If you go librelist.com you can read about it, although
the site is still rough and being written. It actually does function however,
including basic archives.
For people too lazy to click on the link: librelist.com is just a free mailing
list service for open source projects, similar to freenode.org with IRC.
The primary purpose is to give open source projects a mailing list they can use that has:
The code available (it’s in the Lamson source).
No logins, signups, bundled services, captchas, tracking cookies, or other crap getting in your way.
No ads on your emails, ever.
Full access to your archives using rsync.
Easy and smart signup process.
Spam blocking and bounce protection galore (although spam is off right now).
Try The Lists
I’ll have an announcement tomorrow laying out the big things you’ll get, but for now
if you want to track Lamson and participate, as well as test the librelist.com
code, then try subscribing to lamson@librelist.com
and saying hello.
If you’re interested in librelist.com development, then subscribe to meta@librelist.com
and I’ll be sending out announcements related to librelist there.
The Lamson 1.0pre1 Plan
I haven’t needed to touch Lamson much over the last few days, apart from some surgery
on the Python Maildir class to add a safety feature. The code currently in bzr is
also very stable and has been sped up a bit. I’m looking to basically spend tomorrow
auditing the code and writing docs with the goal of pushing out a release in the evening.
The goal then will be to make it solid and not add many more features so that I can get a 1.0
release out for real by the end of July.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-07-19.txt 0000644 0000765 0000024 00000004462 11230540446 023355 0 ustar zedshaw staff Title: I Blame Bounces For The Delay
I was going to release Lamson 0.9.6 much earlier than this, but as I worked on
the bounce detection feature, I realized that if I just wrote that and a few
other cleanups and features, I'd actually have a *1.0pre1* candidate on my
hands instead of a 0.9.6 release.
Instead of releasing 0.9.6, I continued to work on "librelist.com":http://librelist.com/
and got the software smart as hell. While working on librelist.com I managed
to improve Lamson to the point where I'll be comfortable releasing a 1.0pre1
tomorrow.
h2. What's librelist.com?
If you go "librelist.com":http://librelist.com/ you can read about it, although
the site is still rough and being written. It actually does function however,
including basic archives.
For people too lazy to click on the link: librelist.com is just a free mailing
list service for open source projects, similar to freenode.org with IRC.
The primary purpose is to give open source projects a mailing list they can use that has:
* The code available (it's in the Lamson source).
* No logins, signups, bundled services, captchas, tracking cookies, or other crap getting in your way.
* No ads on your emails, ever.
* Full access to your archives using rsync.
* Easy and smart signup process.
* Spam blocking and bounce protection galore (although spam is off right now).
h2. Try The Lists
I'll have an announcement tomorrow laying out the big things you'll get, but for now
if you want to track Lamson and participate, as well as test the librelist.com
code, then try subscribing to "lamson@librelist.com":mailto:lamson@librelist.com
and saying hello.
If you're interested in librelist.com development, then subscribe to "meta@librelist.com":mailto:meta@librelist.com
and I'll be sending out announcements related to librelist there.
h2. The Lamson 1.0pre1 Plan
I haven't needed to touch Lamson much over the last few days, apart from some surgery
on the Python Maildir class to add a safety feature. The code currently in bzr is
also very stable and has been sped up a bit. I'm looking to basically spend tomorrow
auditing the code and writing docs with the goal of pushing out a release in the evening.
The goal then will be to make it solid and not add many more features so that I can get a 1.0
release out for real by the end of July.
See you tomorrow!
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-07-20.html 0000644 0000765 0000024 00000020476 11230770144 023475 0 ustar zedshaw staff
LamsonProject: Lamson 1.0pre1 Released
Tonight I’m releasing Lamson 1.0pre1 with all the latest improvements I’ve made while
making librelist.com and taking feedback from a few people
using Lamson. The goal from now on will be to basically squash bugs and write docs, with
only a rare feature or two as I find them needed on projects.
What’s In This Release
This release features really only four major enhancements, and a bunch of little fixes
here and there.
There’s now a working mailing list example in examples/librelist.
The router now uses the envelope header To (message.To) instead of the headers.
There is now very solid bounce detection and analysis.
Modules can specify anything as the state key (from, to+from, module+to, etc.)
Here’s the important changes from the change log by category:
Enhancements
340 Added a version command finally, with some good information.
330 There is now a safe=False option to lamson.queue.Queue to use a modified Maildir that md5 hashes the original hostname used.
321 Routing it more deterministic and a bit quicker. It now uses the envelope To for routing.
320 Refactored the routing to it is more deterministic and doesn’t loop over the functions mulitple times. Still some work to do though.
314 Bounce information now includes the original messages in easier to access forms.
308 Got bounce handling working and tested.
303 Slight reorg with the way @state_key_generator is implemented.
302 Implemented a means to specify that a module will key off of different parts of a message.
301 Got tired of not seeing errors during test runs so I made an option in the Router to explode on exceptions vs. log them.
300 Bounce routing decorator for doing the actual routing of messages that are bounces.
296 Implemented a bounce analysis method that is simple but seems to work on most bounce mail.
Bug Fixes
329 Found an email that had a nasty header causing an edge case in the parser.
325 Bounce detection was using the wrong email address.
319 Removed the egg dependency on sqlalchemy.
318 Moved the bounce stuff around in order to make it more advanced.
307 Fixed up the @bounce_to decorator so that it returns the next state.
294 Small bug in multipart handling where content types of 'message/’ can be multipart too.
Librelist Related
323 librelist.com sample is looking good, bounce handling is working, archives are started.
322 Super bounce handling for librelist.com.
306 Got the first cut of the whole app.handlers.admin working, including bounces, but Lamson bounce detection has a flaw.
305 Got the basics of making mailing lists, managing subscribers.
304 Initial commit of the librelist.com sample application.
Also, librelist demonstrates how to integrate Lamson with Django which
turns out to just require a couple lines of code. I’ll have some documentation on that soon.
Docs and Website Related
317 Removed references to SQLAlchemy being needed in the docs.
Getting This Release
As usual, try to install it using easy_install
and make sure you do the upgrade:
$ sudo easy_install -U lamson
This release does not depend on SQLAlchemy anymore, since you can really use Lamson with any
data source you can access with Python. As mentioned above, the librelist sample uses
Django models.
If you want to download the source you can visit the releases page and
grab it from there.
Mailing List Back Online
Finally, the lamson mailing list is now back online, so you can subscribe to it
and get help. Read the lists page to find out more.
IRC Channel
I’m also more active in the IRC channel #lamson on irc.freenode.org. Come by
if you need help.
Thanks again folks, and shoot me any bugs you find.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-07-20.txt 0000644 0000765 0000024 00000007546 11230764565 023365 0 ustar zedshaw staff Title: Lamson 1.0pre1 Released
Tonight I'm releasing Lamson 1.0pre1 with all the latest improvements I've made while
making "librelist.com":http://librelist.com/ and taking feedback from a few people
using Lamson. The goal from now on will be to basically squash bugs and write docs, with
only a rare feature or two as I find them needed on projects.
h2. What's In This Release
This release features really only four major enhancements, and a bunch of little fixes
here and there.
# There's now a working mailing list example in examples/librelist.
# The router now uses the envelope header To (message.To) instead of the headers.
# There is now very solid bounce detection and analysis.
# Modules can specify anything as the state key (from, to+from, module+to, etc.)
Here's the important changes from the change log by category:
h3. Enhancements
* 340 Added a version command finally, with some good information.
* 330 There is now a safe=False option to lamson.queue.Queue to use a modified Maildir that md5 hashes the original hostname used.
* 321 Routing it more deterministic and a bit quicker. It now uses the envelope To for routing.
* 320 Refactored the routing to it is more deterministic and doesn't loop over the functions mulitple times. Still some work to do though.
* 314 Bounce information now includes the original messages in easier to access forms.
* 308 Got bounce handling working and tested.
* 303 Slight reorg with the way @state_key_generator is implemented.
* 302 Implemented a means to specify that a module will key off of different parts of a message.
* 301 Got tired of not seeing errors during test runs so I made an option in the Router to explode on exceptions vs. log them.
* 300 Bounce routing decorator for doing the actual routing of messages that are bounces.
* 296 Implemented a bounce analysis method that is simple but seems to work on most bounce mail.
h3. Bug Fixes
* 329 Found an email that had a nasty header causing an edge case in the parser.
* 325 Bounce detection was using the wrong email address.
* 319 Removed the egg dependency on sqlalchemy.
* 318 Moved the bounce stuff around in order to make it more advanced.
* 307 Fixed up the @bounce_to decorator so that it returns the next state.
* 294 Small bug in multipart handling where content types of 'message/' can be multipart too.
h3. Librelist Related
* 323 librelist.com sample is looking good, bounce handling is working, archives are started.
* 322 Super bounce handling for librelist.com.
* 306 Got the first cut of the whole app.handlers.admin working, including bounces, but Lamson bounce detection has a flaw.
* 305 Got the basics of making mailing lists, managing subscribers.
* 304 Initial commit of the librelist.com sample application.
Also, librelist demonstrates how to integrate Lamson with "Django":http://www.djangoproject.com/ which
turns out to just require a couple lines of code. I'll have some documentation on that soon.
h3. Docs and Website Related
* 317 Removed references to SQLAlchemy being needed in the docs.
h2. Getting This Release
As usual, try to install it using "easy_install":http://peak.telecommunity.com/DevCenter/EasyInstall
and make sure you do the upgrade:
$ sudo easy_install -U lamson
This release does not depend on SQLAlchemy anymore, since you can really use Lamson with any
data source you can access with Python. As mentioned above, the librelist sample uses
Django models.
If you want to download the source you can visit the "releases":/releases/ page and
grab it from there.
h2. Mailing List Back Online
Finally, the lamson mailing list is now back online, so you can subscribe to it
and get help. Read "the lists page":/lists/ to find out more.
h2. IRC Channel
I'm also more active in the IRC channel #lamson on irc.freenode.org. Come by
if you need help.
Thanks again folks, and shoot me any bugs you find.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-08-03.html 0000644 0000765 0000024 00000052216 11235577731 023510 0 ustar zedshaw staff
LamsonProject: Lamson 1.0pre2, HTML Email, Standalone
Lamson 1.0pre2 features two features that might signal the end of the beginning or
the beginning of the end, depending on your perspective: HTML Email and Lamson Standalone.
HTML Email support comes from a new module
lamson.html that gives a nice template
method to send out HTML to victims…uh…customers. Lamson Standalone will be a way
to run Lamson as your customized email server instead of another server like Postfix.
Both of these features will be done for Lamson 1.0, but are currently just getting started
as of 1.0pre2. Once they’re done that will be the last two requested features people
had before 1.0.
Getting This Release
As usual, you can get this release from /releases or if you are too lazy to
read that page then do this:
$ sudo easy_install -U lamson
In addition, if you want play with the new features you’ll need to install these (optional)
Python libraries:
I’ll decide if I should make these mandatory or optional with the next release based on
people’s feedback.
Lamson Standalone
Quite a few people have asked for a way to install Lamson as their primary email server.
Usually they want it for just their own email on a machine just for them where installing
something like Postfix is overkill or just too hard. This is entirely possible
with Lamson, but it’s kind of not the right use for it since simple delivery of mail is
much better handled and implemented by an older server like Postfix.
Rather than resist giving people a way to setup Lamson as their primary mail server, I’ve
decided to start making it possible for them. By the next release (1.0pre3) you’ll be able
to run Lamson as your only email server for small installations handling mail for one person.
I’ll also include a screen cast showing people how to use the standalone functionality
to build a Personal Mail Management Server to filter and control their mail without any
other mail server. I’m going to treat this as a first project for most people wanting to get
started using Lamson for their email.
As of 1.0pre2, the functionality that’s available is code in the lamson.server.Relay
that will use PyDNS to query up the MX host for
recipient addresses. It’s fairly primitive right now, but you use it by creating
a Relay with the host explicitly None:
relay = server.Relay(host=None, port=25, debug=1)
Once you do this the Relay will lookup the hosts rather than trying to send through a relay
host.
Don’t go crazy with this yet, since it has to be tested with various kind of nasty email
addressing out there, and it needs to have a way to generate a bounce when it has an error.
HTML Email
Well, I broke down and implemented my idea for making HTML Email easy as hell to generate.
In the past I didn’t want to include simple HTML support because, well, HTML Email is
annoying as hell and I didn’t want to deal with the support headaches. I knew once I threw
HTML generation into Lamson I’d have an army of marketing people using Lamson poorly to
generate their marketing materials. I also think that HTML formatting in email doesn’t work
as a customer development strategy.
Yet, every time I tell someone about Lamson, the very first, second, third, and 300th thing they
ask is if it does HTML Email. Over and over and over this was the most important feature, above
spam blocking, filtering, building applications, intelligent state management, or anything else
that Lamson supports.
Well, if the people want HTML email generation, then the people will get it. I introduce
you to lamson.html which
makes it trivial to produce HTML in your email and doing it in a nice clean way using
CleverCSS and Jinja2
templates.
Let’s start with the simplest little example that will send out a disgusting html template:
import sys
from lamson import html, server
from config import testing
relay = server.Relay(host=None, port=25, debug=1)
hs = html.HtmlMail("style.css", "html_test.html")
title = "Test Message HTML"
msg = hs.respond(locals(), "content.markdown", From=sys.argv[1],
To=sys.argv[2], Subject="Test %(title)s")
relay.deliver(msg)
You could run this right out of the Lamson source tree like this:
The result would look like this in victim’s email:
How lamson.html.HtmlMail Works
The rationale behind the HtmlMail class is that you’ll typically have a template you
want to send, some CSS, and then body content that you’ll plug into the template
as a slug for each person. What HtmlMail does is let you setup the template as HTML
with Jinja2, and then specify your CSS using CleverCSS (which is way easier).
When you go to send, you generate a Markdown
template as the body. This lets you write the actual body of your HTML mail the same
way you would a regular email, but still let’s you get a good HTML output using the
HTML/CSS templates. With markdown regular folks can write the marketing copy for
your spam while keeping the design separated.
HtmlMail then glues this all together by doing the following:
Parses your CleverCSS template (after running it through Jinja2).
Runs your content markdown through Jinja2 and markdown2 just like all your other Lamson templates.
Injects your CSSinto your HTML tags so that you get your styles even though many clients rip out the style tags from your HTML. See the output sample below.
Knits your generated content into your HTML template as the {{content}} variable.
The end result is that these two lines of code from the above sample:
Will let you blast out well formatted HTML emails that have a high probability
of displaying in most mail clients.
How The HTML And CSS Looks
Here’s what each of the files in the above sample look like. First the style.css:
body:
margin: 10
padding: 20
background: green - 30
color: blue
h1:
font-size: 3em
h2:
font-size: 2em
color: yellow
h2:
font-size: 1em
p:
padding: 0.3em
background: red
h2:
color: yellow
#bright:
background: black
color: white
.dull:
background: gray
color: black
Notice that CleverCSS supports quite a few
very cool features, like calling functions, nesting, variables, and calculations.
Next you have the outer template html_template.html that wraps your content markdown template. Notice
that there’s no style tag where the above CleverCSS is placed. That gets done
later by lamson.html.HtmlMail.
<html>
<head>
<title>{{ title }}</title>
</head>
<body style="background: magenta">
<h1 class="bright">{{ title }}</h1>
{{ content }}
<h2 id="dull">All done.</h2>
</body>
</html>
Finally, you have the content.markdown file that has your basic markdown formatted email:
Hello
=====
I would *love* for you to tell me what is going on here joe. NOW!
Alright
-------
This is the best I can come up with.
Zed
You can use any format you want by changing a setting when you construct your HtmlMail object,
but markdown seems to be the closest to what people would send as plain text for their email.
In fact, I’d like to find a way to send the raw markdown as the plaintext version of each HTML
email that goes out.
With the above three components, you then get the following output:
<html>
<head>
<title>Test Message HTML</title>
</head>
<body style="margin: 10; padding: 20; background: #006200; color: blue">
<h1 class="bright" style="background: black; color: white">Test Message HTML</h1>
<h1 style="font-size: 3em">Hello</h1>
<p style="padding: 0.3em; background: red">I would <em>love</em> for you to tell me what is going on here joe. NOW!</p>
<h2 style="color: yellow">Alright</h2>
<p style="padding: 0.3em; background: red">This is the best I can come up with.</p>
<p style="padding: 0.3em; background: red">Zed</p>
<h2 id="dull" style="background: gray; color: black">All done.</h2>
</body>
</html>
Like magic HtmlMail has taken your CleverCSS file and merged it into the tags of your HTML to
style it, making it work in most clients without forcing you to do it manually. It does this by
walking the CSS and generated HTML with BeautifulSoup
and setting the style attribute as it goes.
Librelist JSON Archives
There’s been a ton of work on librelist.com to make it
work better, but the big feature is a complete JSON dump of the mail archives
and a new Archive Browser for reading the
archives.
The browser is just a fast one day hack I did to prove that the JSON
archive format was good. It uses JQuery UI to build
the interface, and has no servers side software other than Nginx. Yes, you
read that right, there is no Django behind this, just a pure Nginx
setup serving files and directory indexes.
The real meat of this setup is the Lamson server on librelist and an Nginx module.
First, the librelist server generates a JSON version of each archived email which
you can see here with your browser. This JSON actually loads as an
object in JavaScript, so you have code like this:
function loadDay(date) {
if(window.MAILING_LIST) {
$.getJSON("/archives/" + window.MAILING_LIST + date + "/json/", {},
function(data) {
$("#messages").empty();
base = data[0]
msgs = data[1]
for(i in msgs) {
fetchMessage(base + msgs[i])
}
});
} else {
$("#messages").html("<h2>Now pick a list to look at for that day.</h2>");
}
}
With those two components I can export a JSON version of the mailing list archives without
having to run a large web framework.
ChangeLog
For those who are interested, here’s the Bazaar logs from this release, separated into
changes to Lamson and changes to the included librelist.com sample.
Lamson Changes
Cleanup and tuning of the html code.
Documenting the HtmlMail API.
New HTML email generation with CleverCSS support and merging the CSS into the HTML results.
Additional tests for the new MX query style of Relay.
Implemented a feature to let the relay do its own delivery, rather than needing a smart host. Requires PyDNS.
Cleaned up the build so that it removes junk from the examples.
Fixed up the lamson default help command so that it is more succinct.
Properly skips spam filtering if the spam databse doesn’t exist.
Made the spam filter not barf if there’s no spam db, which helps when testing.
Now don’t bother storing the START state since it’s the default.
Cleaned up mail some more and deprecated the MailRequest.msg and MailResponse.msg to be replaced with .base (since that’s what it is).
Added a simple method to copy the attachments of a MailRequest to a MailResponse.
Got decent attachment copying implemented, mail api needs a bit of cleanup.
Return an empty string when trying to encode a none.
Minor bug in the safequeue, added logging so you can track oversize.
Getting the headers right for replies to work, removed the first message.
Added options to Queue and QueueReceiver to limit the size of incoming mail.
Added a version command finally, with some good information.
librelist.com Changes
Force the permissions to be correct for serving up the archives.
Make sure that bounces are always saved for later.
Json convert script can’t go there.
Need to install simplejson to make the json work on librelist.
Prevent loop back of messages from lamson to itself, and stop messages that should not be considered lists like unbounce and noreply.
Implements a json version of the mailing list archives in addition to the regular MIME encoded ones.
Update of changes from the live server for setting up the spam system.
Migration to install the spam database on deployment.
Added spam filtering to the admin handler where it counts.
Confirmations are removed after they are verified.
Template for bad list name help.
Tweaked the formatting for the bad list name helper, and made sure that it returns to START.
Now giving out an error message for people who get the name format wrong.
Added additional headers needed for achives, and try to maintain the date and message-id headers.
Production and staging deployment scripts setup and a migration to make those work.
Implemented a simple bash based deployment setup for automating librelist deploys and migrations.
Nope, can’t add the – yet without some regex wizardry.
Small change to allow – in the mailing list names.
Updated the messages librelist sends out for readability.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-08-03.txt 0000644 0000765 0000024 00000035510 11235577723 023362 0 ustar zedshaw staff Title: Lamson 1.0pre2, HTML Email, Standalone
Lamson 1.0pre2 features two features that might signal the end of the beginning or
the beginning of the end, depending on your perspective: HTML Email and Lamson Standalone.
HTML Email support comes from a new module
"lamson.html":http://lamsonproject.org/docs/api/lamson.html-module.html that gives a nice template
method to send out HTML to victims...uh...customers. Lamson Standalone will be a way
to run Lamson as your customized email server instead of another server like Postfix.
Both of these features will be done for Lamson 1.0, but are currently just getting started
as of 1.0pre2. Once they're done that will be the last two requested features people
had before 1.0.
h2. Getting This Release
As usual, you can get this "release from /releases":/releases/ or if you are too lazy to
read that page then do this:
$ sudo easy_install -U lamson
In addition, if you want play with the new features you'll need to install these (optional)
Python libraries:
I'll decide if I should make these mandatory or optional with the next release based on
people's feedback.
h2. Lamson Standalone
Quite a few people have asked for a way to install Lamson as their primary email server.
Usually they want it for just their own email on a machine just for them where installing
something like Postfix is overkill or just too hard. This is entirely possible
with Lamson, but it's kind of not the right use for it since simple delivery of mail is
much better handled and implemented by an older server like Postfix.
Rather than resist giving people a way to setup Lamson as their primary mail server, I've
decided to start making it possible for them. By the next release (1.0pre3) you'll be able
to run Lamson as your only email server for small installations handling mail for one person.
I'll also include a "screen cast":/videos/ showing people how to use the standalone functionality
to build a *Personal Mail Management Server* to filter and control their mail without any
other mail server. I'm going to treat this as a first project for most people wanting to get
started using Lamson for their email.
As of 1.0pre2, the functionality that's available is code in the "lamson.server.Relay":http://lamsonproject.org/docs/api/lamson.server.Relay-class.html
that will use "PyDNS":http://pydns.sourceforge.net/ to query up the MX host for
recipient addresses. It's fairly primitive right now, but you use it by creating
a Relay with the host explicitly None:
relay = server.Relay(host=None, port=25, debug=1)
Once you do this the Relay will lookup the hosts rather than trying to send through a relay
host.
Don't go crazy with this yet, since it has to be tested with various kind of nasty email
addressing out there, and it needs to have a way to generate a bounce when it has an error.
h2. HTML Email
Well, I broke down and implemented my idea for making HTML Email easy as hell to generate.
In the past I didn't want to include simple HTML support because, well, HTML Email is
annoying as hell and I didn't want to deal with the support headaches. I knew once I threw
HTML generation into Lamson I'd have an army of marketing people using Lamson poorly to
generate their marketing materials. I also think that HTML formatting in email doesn't work
as a customer development strategy.
Yet, every time I tell someone about Lamson, the very first, second, third, and 300th thing they
ask is if it does HTML Email. Over and over and over this was the most important feature, above
spam blocking, filtering, building applications, intelligent state management, or anything else
that Lamson supports.
Well, if the people want HTML email generation, then the people will get it. I introduce
you to "lamson.html":http://lamsonproject.org/docs/api/lamson.html-module.html which
makes it trivial to produce HTML in your email and doing it in a nice clean way using
"CleverCSS":http://sandbox.pocoo.org/clevercss/ and "Jinja2":http://jinja.pocoo.org/2/
templates.
Let's start with the simplest little example that will send out a disgusting html template:
import sys
from lamson import html, server
from config import testing
relay = server.Relay(host=None, port=25, debug=1)
hs = html.HtmlMail("style.css", "html_test.html")
title = "Test Message HTML"
msg = hs.respond(locals(), "content.markdown", From=sys.argv[1],
To=sys.argv[2], Subject="Test %(title)s")
relay.deliver(msg)
You could run this right out of the Lamson source tree like this:
The result would look like this in victim's email:
!html_email_in_gmail.png!
h2. How lamson.html.HtmlMail Works
The rationale behind the HtmlMail class is that you'll typically have a template you
want to send, some CSS, and then body content that you'll plug into the template
as a slug for each person. What HtmlMail does is let you setup the template as HTML
with Jinja2, and then specify your CSS using CleverCSS (which is way easier).
When you go to send, you generate a "Markdown":http://daringfireball.net/projects/markdown/
template as the body. This lets you write the actual body of your HTML mail the same
way you would a regular email, but still let's you get a good HTML output using the
HTML/CSS templates. With markdown regular folks can write the marketing copy for
your spam while keeping the design separated.
HtmlMail then glues this all together by doing the following:
# Parses your CleverCSS template (after running it through Jinja2).
# Runs your content markdown through Jinja2 and "markdown2":http://code.google.com/p/python-markdown2/ just like all your other Lamson templates.
# Injects your CSS *into* your HTML tags so that you get your styles even though many clients rip out the *style* tags from your HTML. See the output sample below.
# Knits your generated content into your HTML template as the {{content}} variable.
The end result is that these two lines of code from the above sample:
Will let you blast out well formatted HTML emails that have a high probability
of displaying in most mail clients.
h2. How The HTML And CSS Looks
Here's what each of the files in the above sample look like. First the @style.css@:
body:
margin: 10
padding: 20
background: green - 30
color: blue
h1:
font-size: 3em
h2:
font-size: 2em
color: yellow
h2:
font-size: 1em
p:
padding: 0.3em
background: red
h2:
color: yellow
#bright:
background: black
color: white
.dull:
background: gray
color: black
Notice that "CleverCSS":http://sandbox.pocoo.org/clevercss/ supports quite a few
very cool features, like calling functions, nesting, variables, and calculations.
Next you have the outer template @html_template.html@ that wraps your content markdown template. Notice
that there's no *style* tag where the above CleverCSS is placed. That gets done
later by lamson.html.HtmlMail.
<html>
<head>
<title>{{ title }}</title>
</head>
<body style="background: magenta">
<h1 class="bright">{{ title }}</h1>
{{ content }}
<h2 id="dull">All done.</h2>
</body>
</html>
Finally, you have the @content.markdown@ file that has your basic markdown formatted email:
Hello
=====
I would *love* for you to tell me what is going on here joe. NOW!
Alright
-------
This is the best I can come up with.
Zed
You can use any format you want by changing a setting when you construct your HtmlMail object,
but markdown seems to be the closest to what people would send as plain text for their email.
In fact, I'd like to find a way to send the raw markdown as the plaintext version of each HTML
email that goes out.
With the above three components, you then get the following output:
<html>
<head>
<title>Test Message HTML</title>
</head>
<body style="margin: 10; padding: 20; background: #006200; color: blue">
<h1 class="bright" style="background: black; color: white">Test Message HTML</h1>
<h1 style="font-size: 3em">Hello</h1>
<p style="padding: 0.3em; background: red">I would <em>love</em> for you to tell me what is going on here joe. NOW!</p>
<h2 style="color: yellow">Alright</h2>
<p style="padding: 0.3em; background: red">This is the best I can come up with.</p>
<p style="padding: 0.3em; background: red">Zed</p>
<h2 id="dull" style="background: gray; color: black">All done.</h2>
</body>
</html>
Like magic HtmlMail has taken your CleverCSS file and merged it into the tags of your HTML to
style it, making it work in most clients without forcing you to do it manually. It does this by
walking the CSS and generated HTML with "BeautifulSoup":http://www.crummy.com/software/BeautifulSoup/
and setting the *style* attribute as it goes.
h2. Librelist JSON Archives
There's been a ton of work on "librelist.com":http://librelist.com/ to make it
work better, but the big feature is a complete JSON dump of the mail archives
and a new "Archive Browser":http://librelist.com/browser/ for reading the
archives.
The browser is just a fast one day hack I did to prove that the JSON
archive format was good. It uses "JQuery UI":http://jqueryui.com/ to build
the interface, and has *no servers side software other than Nginx.* Yes, you
read that right, there is no Django behind this, just a pure "Nginx":http://nginx.net/
setup serving files and directory indexes.
The real meat of this setup is the Lamson server on librelist and an Nginx module.
First, the librelist server generates a JSON version of each archived email which
you "can see here":http://librelist.com/archives/lamson/2009/07/30/json/1248947932.M724307P15430Q6.09c5769d5b9f3d575cefc2ccb51877ec.json with your browser. This JSON actually loads as an
object in JavaScript, so you have code like this:
Second, there's a tiny modification to Nginx's
"autoindex":http://wiki.nginx.org/NginxHttpAutoindexModule I made so that
it will generate a "JSON version of the index instead of HTML":http://librelist.com/archives/lamson/2009/07/
so that you can "browse" the stored JSON files via JavaScript:
function loadDay(date) {
if(window.MAILING_LIST) {
$.getJSON("/archives/" + window.MAILING_LIST + date + "/json/", {},
function(data) {
$("#messages").empty();
base = data[0]
msgs = data[1]
for(i in msgs) {
fetchMessage(base + msgs[i])
}
});
} else {
$("#messages").html("<h2>Now pick a list to look at for that day.</h2>");
}
}
With those two components I can export a JSON version of the mailing list archives without
having to run a large web framework.
h2. ChangeLog
For those who are interested, here's the Bazaar logs from this release, separated into
changes to Lamson and changes to the included librelist.com sample.
h2. Lamson Changes
* Cleanup and tuning of the html code.
* Documenting the HtmlMail API.
* New HTML email generation with CleverCSS support and merging the CSS into the HTML results.
* Additional tests for the new MX query style of Relay.
* Implemented a feature to let the relay do its own delivery, rather than needing a smart host. Requires PyDNS.
* Cleaned up the build so that it removes junk from the examples.
* Fixed up the lamson default help command so that it is more succinct.
* Properly skips spam filtering if the spam databse doesn't exist.
* Made the spam filter not barf if there's no spam db, which helps when testing.
* Now don't bother storing the START state since it's the default.
* Cleaned up mail some more and deprecated the MailRequest.msg and MailResponse.msg to be replaced with .base (since that's what it is).
* Added a simple method to copy the attachments of a MailRequest to a MailResponse.
* Got decent attachment copying implemented, mail api needs a bit of cleanup.
* Return an empty string when trying to encode a none.
* Minor bug in the safequeue, added logging so you can track oversize.
* Getting the headers right for replies to work, removed the first message.
* Added options to Queue and QueueReceiver to limit the size of incoming mail.
* Added a version command finally, with some good information.
h2. librelist.com Changes
* Force the permissions to be correct for serving up the archives.
* Make sure that bounces are always saved for later.
* Json convert script can't go there.
* Need to install simplejson to make the json work on librelist.
* Prevent loop back of messages from lamson to itself, and stop messages that should not be considered lists like unbounce and noreply.
* Implements a json version of the mailing list archives in addition to the regular MIME encoded ones.
* Update of changes from the live server for setting up the spam system.
* Migration to install the spam database on deployment.
* Added spam filtering to the admin handler where it counts.
* Confirmations are removed after they are verified.
* Template for bad list name help.
* Tweaked the formatting for the bad list name helper, and made sure that it returns to START.
* Now giving out an error message for people who get the name format wrong.
* Added additional headers needed for achives, and try to maintain the date and message-id headers.
* Production and staging deployment scripts setup and a migration to make those work.
* Implemented a simple bash based deployment setup for automating librelist deploys and migrations.
* Nope, can't add the - yet without some regex wizardry.
* Small change to allow - in the mailing list names.
* Updated the messages librelist sends out for readability.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-08-22.html 0000644 0000765 0000024 00000023450 11243717167 023505 0 ustar zedshaw staff
LamsonProject: Lamson 1.0pre4 Out, Lots Of Docs Done, 100% Coverage
Lamson 1.0pre4 Out, Lots Of Docs Done, 100% Coverage
I happy to announce probably one of the last few releases before
I officially put the 1.0 stamp on Lamson. This last 1% of the things
I want to do takes a while, but it really puts a good shine on the
project.
What I’ve got in 1.0pre4 is tons of docs
and 100% test coverage. Most of the features that were added are
nice-to-haves that I’ve found useful while developing my various sites.
Getting The Release
As usual, sudo easy_install lamson is your friend. If you want
to use the HTML generation features, then you’ll want to also install:
BeautifulSoup
CleverCSS
markdown2
For completeness, here’s the remaining packages I have installed in most
of the virtualenvs for my applications:
It’s mostly a vanity thing, but doing this did uncover a couple of
minor little bugs here and there, and it makes people feel better
trusting Lamson.
Docs, Docs, Docs
I firmly believe that a good software project has bothAPI
style documentation and guided documentation. That’s why I spent almost this whole
time working on documenting all the parts of Lamson people need and making sure the generated
docs were complete.
What I’ve done is reorganized the documentation section so that you can find
topic by their categories, and then documented most of the features people
need to use daily. Here’s some highlights:
I especially like how the HTML Email generation came out as a feature, but I really
like the Bounce Detection
documentation the most.
New Confirmation API
This release has a few little bug fixes, but mostly it has a solidified feature for
doing Confirmations easily.
HTML/Text Dual Email
Previously the HTML Email Generation
feature could only generate HTML email reliably, but I managed to track down
a bug that lets you now craft dual email. I even went a step further and made it easy
to simply use your markdown content templates as your text/plain alternative
when you send out HTML Email.
Multiple Recipient Routing
There was no way to avoid this, so I just implemented it. Now when a message is for
multiple recipients (in the envelope) Lamson will route the message to each
address. That means if it is destined for multiple people, then Lamson will
effectively try to match each of those people and run them through your handlers
separately.
I was worried this would cause problems, but so far it’s working fine. If you see
that it can be used as a DOS attack, then I’ll work on a way to limit the number
of recipients that Lamson will allow (similar to how you can restrict the size
of messages off a Queue).
Test And Report
As usual, please test this out, review the documentation, and report back to me
any problems you find.
I’ll be in #lamson on irc.freenode.org more, and will probably install a IRC
bot to notify when I post new code to the repository.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-08-22.txt 0000644 0000765 0000024 00000012217 11243717140 023346 0 ustar zedshaw staff Title: Lamson 1.0pre4 Out, Lots Of Docs Done, 100% Coverage
I happy to announce probably one of the last few releases before
I officially put the 1.0 stamp on Lamson. This last 1% of the things
I want to do takes a while, but it really puts a good shine on the
project.
What I've got in 1.0pre4 is "tons of docs":http://lamsonproject.org/docs/
and 100% test coverage. Most of the features that were added are
nice-to-haves that I've found useful while developing my various sites.
h2. Getting The Release
As usual, @sudo easy_install lamson@ is your friend. If you want
to use the HTML generation features, then you'll want to also install:
BeautifulSoup
CleverCSS
markdown2
For completeness, here's the remaining packages I have installed in most
of the virtualenvs for my applications:
You can also grab "source releases":/releases/ and "download instructions":/download.html
from this site if PyPI is failing you.
h2. 100% Test Coverage
First, I've managed to get the test coverage for Lamson up to 100%:
It's mostly a vanity thing, but doing this did uncover a couple of
minor little bugs here and there, and it makes people feel better
trusting Lamson.
h2. Docs, Docs, Docs
I firmly believe that a good software project has *both* "API":http://lamsonproject.org/docs/api/lamson-module.html
style documentation and guided documentation. That's why I spent almost this whole
time working on documenting all the parts of Lamson people need and making sure the generated
docs were complete.
What I've done is reorganized the documentation section so that you can find
topic by their categories, and then documented most of the features people
need to use daily. Here's some highlights:
* "Unit Testing":http://lamsonproject.org/docs/unit_testing.html
* "Confirmations":http://lamsonproject.org/docs/confirmations.html
* "Filtering Spam":http://lamsonproject.org/docs/filtering_spam.html
* "Bounce Detection":http://lamsonproject.org/docs/bounce_detection.html
* "HTML Email Generation":http://lamsonproject.org/docs/html_email_generation.html
* "Unicode Encoding/Decoding":http://lamsonproject.org/docs/unicode_encoding_and_decoding.html
* "Hooking Into Django":http://lamsonproject.org/docs/hooking_into_django.html
* "Tons more...":http://lamsonproject.org/docs/
I especially like how the HTML Email generation came out as a feature, but I really
like the "Bounce Detection":http://lamsonproject.org/docs/bounce_detection.html
documentation the most.
h2. New Confirmation API
This release has a few little bug fixes, but mostly it has a solidified feature for
doing "Confirmations":http://lamsonproject.org/docs/confirmations.html easily.
h2. HTML/Text Dual Email
Previously the "HTML Email Generation":http://lamsonproject.org/docs/html_email_generation.html
feature could only generate HTML email reliably, but I managed to track down
a bug that lets you now craft dual email. I even went a step further and made it easy
to simply use your markdown content templates as your text/plain alternative
when you send out HTML Email.
h2. Multiple Recipient Routing
There was no way to avoid this, so I just implemented it. Now when a message is for
multiple recipients (in the envelope) Lamson will route the message to each
address. That means if it is destined for multiple people, then Lamson will
effectively try to match each of those people and run them through your handlers
separately.
I was worried this would cause problems, but so far it's working fine. If you see
that it can be used as a DOS attack, then I'll work on a way to limit the number
of recipients that Lamson will allow (similar to how you can restrict the size
of messages off a Queue).
h2. Test And Report
As usual, please test this out, review the documentation, and report back to me
any problems you find.
I'll be in #lamson on irc.freenode.org more, and will probably install a IRC
bot to notify when I post new code to the repository.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-09-07.html 0000644 0000765 0000024 00000012374 11251323150 023474 0 ustar zedshaw staff
LamsonProject: Lamson 1.0pre5 Out
You can also get it from releases or by doing the usual
easy_install.
This is a small bugfix release that only adds one small feature. If
you’re running with the SMTPReceiver, you can raise a server.SMTPError with
an error code you want and the SMTPReceiver will return that as the error
code to the client.
Then it will generate a reasonable error message (using lamson.bounce
messages) and give that to the client. You can also give a second
parameter to SMTPError that will be the error message if you don’t
like the automatic ones.
Only warning with this is use it sparingly, as it then ties you always
running Lamson as an SMTPReceiver, instead of off the QueueReceiver. It also
leaks out to malicious clients more information about your server than
you might want them to know. In general, it’s better to just ignore bad
messages and save them in an undeliverable queue for later inspection.
Alright, try it out and report bugs to me. Thanks.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-09-07.txt 0000644 0000765 0000024 00000002417 11251323145 023350 0 ustar zedshaw staff Title: Lamson 1.0pre5 Out
I just pushed Lamson 1.0pre5 to PyPI for your enjoyment:
"PyPI Download":http://pypi.python.org/pypi?:action=files&name=lamson&version=1.0pre5
You can also get it from "releases":/releases/ or by doing the usual
easy_install.
This is a small bugfix release that only adds one small feature. If
you're running with the SMTPReceiver, you can raise a server.SMTPError with
an error code you want and the SMTPReceiver will return that as the error
code to the client.
For example, if you do this in your handler:
Then it will generate a reasonable error message (using lamson.bounce
messages) and give that to the client. You can also give a second
parameter to SMTPError that will be the error message if you don't
like the automatic ones.
Only warning with this is use it sparingly, as it then ties you always
running Lamson as an SMTPReceiver, instead of off the QueueReceiver. It also
leaks out to malicious clients more information about your server than
you might want them to know. In general, it's better to just ignore bad
messages and save them in an undeliverable queue for later inspection.
Alright, try it out and report bugs to me. Thanks.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-09-26.html 0000644 0000765 0000024 00000012361 11257445556 023516 0 ustar zedshaw staff
LamsonProject: Quick Lamson 1.0pre6 Release, More This Weekend
I put up a quick release of Lamson that fixes a bug in the gen project prototype.
I recently switched all the domain names being used in the test suite to “localhost” or
a similar nonexistant domain name. Why? Well, because some people liked to run
the Lamson test suite or their fresh project test suite against a liveSMTP
server. The test suite would “send” mail to test@test.com and my own
address, under the assumption that people would use the lamson log test server.
End result: Periodically I’d get bounce messages from test.com and other test
suite mail.
Well, I made that change, but forgot to update a couple places in the prototype,
so newbies were seeing the tests for their fresh project fail and wondering
if Lamson was broken.
Sorry about that.
You can grab this 1.0pre6 release from the releases area, and
you can also do the usual easy_install -U lamson to get it.
I’ll be doing some more work to fix a few bugs this weekend and working on a
fun new site for guitarists, so stay tuned. And if there’s something in particular
you want in Lamson just email the lamson@librelist.com mailing list.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-09-26.txt 0000644 0000765 0000024 00000002264 11257445456 023371 0 ustar zedshaw staff Title: Quick Lamson 1.0pre6 Release, More This Weekend
I put up a quick release of Lamson that fixes a bug in the gen project prototype.
I recently switched all the domain names being used in the test suite to "localhost" or
a similar nonexistant domain name. Why? Well, because some people liked to run
the Lamson test suite or their fresh project test suite against a *live* SMTP
server. The test suite would "send" mail to test@test.com and my own
address, under the assumption that people would use the @lamson log@ test server.
End result: Periodically I'd get bounce messages from test.com and other test
suite mail.
Well, I made that change, but forgot to update a couple places in the prototype,
so newbies were seeing the tests for their fresh project fail and wondering
if Lamson was broken.
Sorry about that.
You can grab this 1.0pre6 release from the "releases":/releases/ area, and
you can also do the usual @easy_install -U lamson@ to get it.
I'll be doing some more work to fix a few bugs this weekend and working on a
fun new site for guitarists, so stay tuned. And if there's something in particular
you want in Lamson just email the lamson@librelist.com mailing list.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-12-12.html 0000644 0000765 0000024 00000020025 11311017436 023456 0 ustar zedshaw staff
LamsonProject: Lamson 1.0pre9 Is Out
UPDATE: This used to be an announce for 1.0pre8, but python-daemon apparently has a bug that makes it not honor
the TERM signal (still) so I had to go back to HUP.
I just got done knocking down all the bugs in the new Fossil based
Lamson Support Site and cooked up 1.0pre9. This has a
ton of fixes, but I’ll let the ChangeLog speak for itself:
=== 2009-12-12 ===
22:49:43 [c2be9dedc3] *CURRENT* Version bump to 1.0pre9 thanks to python-
daemon. (user: zedshaw tags: trunk, 1.0pre9)
22:43:04 [1718d014a5] Stupid fucking python-daemon doesn't recognize TERM yet.
(user: zedshaw tags: trunk)
20:24:20 [b5796e14a1] Pointed at the tag instead of just the leaves. (user:
zedshaw tags: trunk)
20:00:38 [42768ade2b] 1.0pre8 release announced and done. (user: zedshaw tags:
trunk, 1.0pre8)
19:43:53 [bb1b0a0dae] Updated MailBase.__nonzero__ to include parts as well.
(user: zedshaw tags: trunk)
19:40:51 [2916b83307] Created ChangeLog from fossil timeline of commits.
(user: zedshaw tags: trunk)
19:36:14 [41530d8a34] Version bump to 1.0pre8 for release. (user: zedshaw
tags: trunk)
19:31:21 [dfe158b282] Initial support of using Delivered-To or similar more
exact To header. (user: zedshaw tags: trunk)
19:14:13 [e9988679b7] Updated prototype.zip for upcoming release. (user:
zedshaw tags: trunk)
19:13:51 [98e94b6103] Potentially fixed the unicode/ascii issues with shelve
storage of confirmations and routing. (user: zedshaw tags: trunk)
18:56:35 [6224ae68e4] Updated docs about confirmation to fix a doc bug. (user:
zedshaw tags: trunk)
18:56:14 [103454c43c] Make things clean up better on INT or TERM (not HUP).
(user: zedshaw tags: trunk)
18:25:29 [009451cabc] Added meta tags to the site so it can be found easier. I
didn't know search engines still use that crap. (user: zedshaw tags:
trunk)
18:22:24 [8908d7852d] Applied starttls, username, password patch to server.
(user: zedshaw tags: trunk)
18:16:01 [9a9f9f37b6] Applied patch for handling odd email addresses and white-
listing what headers have email addresses. (user: zedshaw tags: trunk)
18:06:06 [7605ef0517] Patch for queue bugs applied [bug 9d330c8f80] (user:
zedshaw tags: trunk)
18:02:50 [ef9808c4f7] Point at the right archive browser. (user: zedshaw tags:
trunk)
18:01:39 [3b3dc9391a] Better logging message on failure to connect to relay.
(user: zedshaw tags: trunk)
=== 2009-12-05 ===
18:05:40 [ddc4369b1d] Initial commit to the new fossil based support site.
(user: zedshaw tags: trunk)
17:48:22 [8140fcad03] initial empty check-in (user: lamson tags: trunk)
This closed out a bunch of bugs, so if you entered a bug in then go make sure that it’s actually working for you.
Getting This Release
As usual, you should be able to install it with PIP or easy_install, and you can get the raw downloads from
the releases section of the site.
Submitting Bugs
Keep in mind that this isn’t as heavily tested as other releases, so if you hit a bug, report it
and I’ll fix it. Even better, if you supply a patch then I’ll consider giving you commit rights.
Potential Contributors
Most of the changes were from supplied patches, and if you supplied a patch and want to contribute
to Lamson, then contact me and I’ll give you an account on the site so you can commit directly.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-12-12.txt 0000644 0000765 0000024 00000007011 11311017422 023324 0 ustar zedshaw staff Title: Lamson 1.0pre9 Is Out
*UPDATE*: This used to be an announce for 1.0pre8, but python-daemon apparently has a bug that makes it not honor
the TERM signal (still) so I had to go back to HUP.
I just got done knocking down all the bugs in the new "Fossil":http://fossil-scm.org/ based
"Lamson Support Site":http://support.lamsonproject.org/ and cooked up 1.0pre9. This has a
ton of fixes, but I'll let the ChangeLog speak for itself:
=== 2009-12-12 ===
22:49:43 [c2be9dedc3] *CURRENT* Version bump to 1.0pre9 thanks to python-
daemon. (user: zedshaw tags: trunk, 1.0pre9)
22:43:04 [1718d014a5] Stupid fucking python-daemon doesn't recognize TERM yet.
(user: zedshaw tags: trunk)
20:24:20 [b5796e14a1] Pointed at the tag instead of just the leaves. (user:
zedshaw tags: trunk)
20:00:38 [42768ade2b] 1.0pre8 release announced and done. (user: zedshaw tags:
trunk, 1.0pre8)
19:43:53 [bb1b0a0dae] Updated MailBase.__nonzero__ to include parts as well.
(user: zedshaw tags: trunk)
19:40:51 [2916b83307] Created ChangeLog from fossil timeline of commits.
(user: zedshaw tags: trunk)
19:36:14 [41530d8a34] Version bump to 1.0pre8 for release. (user: zedshaw
tags: trunk)
19:31:21 [dfe158b282] Initial support of using Delivered-To or similar more
exact To header. (user: zedshaw tags: trunk)
19:14:13 [e9988679b7] Updated prototype.zip for upcoming release. (user:
zedshaw tags: trunk)
19:13:51 [98e94b6103] Potentially fixed the unicode/ascii issues with shelve
storage of confirmations and routing. (user: zedshaw tags: trunk)
18:56:35 [6224ae68e4] Updated docs about confirmation to fix a doc bug. (user:
zedshaw tags: trunk)
18:56:14 [103454c43c] Make things clean up better on INT or TERM (not HUP).
(user: zedshaw tags: trunk)
18:25:29 [009451cabc] Added meta tags to the site so it can be found easier. I
didn't know search engines still use that crap. (user: zedshaw tags:
trunk)
18:22:24 [8908d7852d] Applied starttls, username, password patch to server.
(user: zedshaw tags: trunk)
18:16:01 [9a9f9f37b6] Applied patch for handling odd email addresses and white-
listing what headers have email addresses. (user: zedshaw tags: trunk)
18:06:06 [7605ef0517] Patch for queue bugs applied [bug 9d330c8f80] (user:
zedshaw tags: trunk)
18:02:50 [ef9808c4f7] Point at the right archive browser. (user: zedshaw tags:
trunk)
18:01:39 [3b3dc9391a] Better logging message on failure to connect to relay.
(user: zedshaw tags: trunk)
=== 2009-12-05 ===
18:05:40 [ddc4369b1d] Initial commit to the new fossil based support site.
(user: zedshaw tags: trunk)
17:48:22 [8140fcad03] initial empty check-in (user: lamson tags: trunk)
This closed out a bunch of bugs, so if you entered a bug in then go make sure that it's actually working for you.
h2. Getting This Release
As usual, you should be able to install it with PIP or easy_install, and you can get the raw downloads from
the "releases":/releases/ section of the site.
h2. Submitting Bugs
Keep in mind that this isn't as heavily tested as other releases, so if you hit a bug, "report it":http://support.lamsonproject.org/
and I'll fix it. Even better, if you supply a patch then I'll consider giving you commit rights.
h2. Potential Contributors
Most of the changes were from supplied patches, and if you supplied a patch and want to contribute
to Lamson, then contact me and I'll give you an account on the site so you can commit directly.
lamson-1.0pre11/doc/lamsonproject.org/output/blog/2009-12-13.html 0000644 0000765 0000024 00000011111 11311261740 023452 0 ustar zedshaw staff
LamsonProject: Small Lamson 1.0pre10 Update
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
UPDATE: This used to be an announce for 1.0pre8, but python-daemon apparently has a bug that makes it not honor
the TERM signal (still) so I had to go back to HUP.
2009-09-26 : Quick Lamson 1.0pre6 Release, More This Weekend
I put up a quick release of Lamson that fixes a bug in the gen project prototype.
I recently switched all the domain names being used in the test suite to “localhost” or
a similar nonexistant domain name. Why? Well, because some people liked to run
the Lamson test suite or their fresh project test suite against a liveSMTP
server. The test suite would “send” mail to test@test.com and my own
address, under the assumption that people would use the lamson log test server.
I happy to announce probably one of the last few releases before
I officially put the 1.0 stamp on Lamson. This last 1% of the things
I want to do takes a while, but it really puts a good shine on the
project.
2009-08-03 : Lamson 1.0pre2, HTML Email, Standalone
Lamson 1.0pre2 features two features that might signal the end of the beginning or
the beginning of the end, depending on your perspective: HTML Email and Lamson Standalone.
HTML Email support comes from a new module
lamson.html that gives a nice template
method to send out HTML to victims…uh…customers. Lamson Standalone will be a way
to run Lamson as your customized email server instead of another server like Postfix.
Tonight I’m releasing Lamson 1.0pre1 with all the latest improvements I’ve made while
making librelist.com and taking feedback from a few people
using Lamson. The goal from now on will be to basically squash bugs and write docs, with
only a rare feature or two as I find them needed on projects.
I was going to release Lamson 0.9.6 much earlier than this, but as I worked on
the bounce detection feature, I realized that if I just wrote that and a few
other cleanups and features, I’d actually have a 1.0pre1 candidate on my
hands instead of a 0.9.6 release.
I think I’m at a point where bounce detection and handling works really well and is
ready for release. I’ve got it running on the latest Lamson demo librelist.com
and it works. I’ll be pushing out a new release of Libre List in a little while
that does some advanced bounce handling, and if that all works, then I’ll push
out 0.9.6.
I just finished committing 0.9.6 code for doing bounce detection and analysis
with Lamson. It’s part of the new mailing list example I’m coding up for the
0.9.6 release which I’ll be running on a free mailing list site I’m going
to release soon. In this blog post I’d like to go through the bounce detection
algorithm and get some feedback and samples from people. So far it works great for
the samples I have, but I want it to be fairly bullet proof.
I’m making up the release of Lamson and doing some server maintenance so I took down the
Lamson server running the mailing lists. I’ll let everyone know when they’re back up.
I just released Lamson 0.9.5 with all the major Unicode refactoring done and working.
This is an important release because 0.9.5 is where I declare that I’m pushing to a 1.0
release and the base Lamson APIs won’t change under penalty of death. In fact they can’t
change because I’m using them myself in a few applications.
2009-06-22 : 0.9.5 Almost There, But Stumped On Templates
Since the 0.9.4 release I’ve rewritten the main part of the decoding parser
so that it’s much cleaner and handles more edge conditions. If there’s one
word that defines what makes MIME horrible it would be “edge”.
2009-06-20 : Lamson 0.9.4 With Unicode Super Powers
Lamson 0.9.4 is out and it’s sporting a completely rewritten and meticulously
crafted encoding system. With the new lamson.encoding code Lamson can now decode nearly any nasty horrible encoded spam
or mail you hand it, turn it into pristine nice Python unicode strings, and
then output sweet clean ascii or utf-8 in a consistent way.
I’m currently polishing off the two final features before I start going for
the Lamson 1.0 release. I’ve been using Lamson to make a few little cute applications
and create one thing for a potential client, and so far I haven’t had to change much
since 0.9.3. It’s great so far and I hope that Lamson 1.0 will be a fun release.
2009-06-09 : Lamson At NYLUG Python Workshop Today @ 6:00PM
Just a quick reminder that I’ll be presenting Lamson to the
NYLUG Python Workshop today at 6:00PM. The event is at
the NY Public Library Hudson Park Branch, 66 Leroy St., NY NY 10014
in NYC and you can find out more here.
2009-06-08 : A Screencast And Docs On Deploying Lamson And OneShotBlog
I just finished writing some new documentation on Deploying Lamson and OneShotBlog that
shows you how to install Lamson and all required software into a completely clean Python 2.6 and virtualenv configuration.
The instructions take you from nothing to a running oneshotblog.com installation that you can
play with and hack on. The process of doing Lamson deployments is still a little too rough for me, but these instructions
should get people started.
This release is the result of me working on my little
oneshotblog.com project while tweaking and refining
Lamson as I go. The end result is 0.9.3 didn’t have a lot of big code changes,
but all the tiny little changes add up to a very nice release. The highlights
of this release are more secure server runs, better character encoding handling
for headers, various cleanups in how mail is queued, and fixes for Python 2.6
support.
I finally got off my ass and put the OneShotBlog sample up.
This is the code (plus a few little tweaks) from the sample that is in the Lamson source
running on another server. It’s using all the features of Lamson, include the new Queue Receiver
functionality. So far it’s working great considering I’ve just been hacking on it on the side to try
out the usability of the Lamson APIs and not really taken it seriously.
The 0.9.2 release is out and ready for everyone to easy_install. I spent the
day getting rid of my tech debt by boosting the Lamson test coverage to a
whopping 97%.
I just pushed Lamson 0.9 up to PyPI for everyone to grab and break. This release
features a complete redesign of the routing, state handling, templating, and a full
set of very complete documentation. Everyone who was using 0.8.x series should be able
to migrate to this version with some work, but it won’t be terribly painful (assuming
you have unit tests).
I have been working hard on the documentation and scrubbing the code for Lamson and the 0.9 release
coming out soon. The only things I feel I need to do before an official 0.9 release are:
First off, my apologies to everyone if your RSS reader went crazy today.
I include documentation changes in the RSS feed so that people can
easily track updates to the Lamson docs. However, that means when I’m
writing a lot of documentation it hits the feed repeatedly.
I've been hard at work cooking up the very nice new routing system, and
I must say it is rather tasty. I've gone and created a whole new
routing and state management design that uses decorators right in your
handler modules to indicate how each state will expect mail addresses.
I wrote a blog post about project ideas for Lamson
on my personal blog. Head on over if you’re looking for something to hack on, or just
want something to read that isn’t about the web.
This is just a quick update to say thanks to Ken Keiter
for creating a new lamsonproject.org site layout and
design. The new site should be easier to read, have more breathing room, and look
easier on the eyes. It’s even got a logo:
A few announcements from my work on Lamson the last few days. I managed to fix
a bug, put Lamson to work doing Lamson’s mailing lists, and use Lamson to do
some spam blocking on my own email account.
Today I launched the Lamson Project site at
lamsonproject.org and started filling in the
content. Lamson is really turning into a fun and useful project, and hopefully
the site will get other people interested in it and using it.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
There are mailing lists available (running Lamson of course) that you can join
to get help. Keep in mind that these lists are running Lamson, and since Lamson is still a
work in progress they might be faulty.
The best way to get help with Lamson is to email me directly. You can send
mail to zedshaw@zedshaw.com to ask for help.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
Lamson supports intelligent bounce detection with its
lamson.bounce
module. The bounce handling is based on a probability that, depending on found
headers, the message is a bounce. It then gives you a nice clean interface to
check who it’s from, the originating SMTP server, the error messages, and any
human readable messages.
How Bounces Actually Work
Figuring out how a bounce is actually handled is a bit difficult because the
majority of the information available is written by people who know very little
of SMTP server operations. When the average programmer thinks of handling a
bounce, she usually has one of these concepts in mind:
The message could not be delivered, so the remote SMTP server sent back a bounce message.
The message could not be delivered, so the local SMTP server sent back a bounce message.
The message could not be delivered, so the recipient’s email client sent back a bounce message.
The message could not be delivered, so Lamson crafted a bounce message.
Obviously logically you can’t have the recipient’s email client send you a
bounce unless the user does something weird (and incredibly annoying). This
makes sense because the message was not delivered. How can the email client
send back a bounce if they don’t receive the message.
Yes, some clients do support sending bounces, but very few people use this
feature. If you do please talk about it as one more edge case to deal with.
Next Lamson can’t craft the bounce messages for you because Lamson is simply
trying to deliver outgoing mail to a smart-host. Lamson does nothing but ask
the smart-host (your local SMTP server) to deliver, and then waits for a
response. That means Lamson is not sending you any kind of bounce unless you
write code to make it do that.
That leaves only two options for who is really sending the bounce message:
the remote MTA or the local MTA. The truth is it’s a little bit of both.
Your Local MTA Is A Person
How a bounce works in practice involves two SMTP servers: your local
smart-host, and the remote server that it tries to connect to for delivery.
The message you actually get in lamson is from the local server, and usually
an address at that local server. It does not come from the remote server,
but inside your bounce will be a message and status indicators from the remote
server indicating why it bounced.
What happens is your local server attempts to deliver, and the remote server
rejects it for some reason. Now your local server is supposed to try again in
certain situations, but after a certain number of rejections or failures it
crafts a bounce message. That bounce message is then returned to your lamson
server based on the envelope header of the message (more on that later).
Now, what’s inside this message? Well, it’s a oddly nested barely standardized
pile of random other messages. This is the frustration with bounce handling.
You pretty much either have to use a probability mechanism (like Lamson does)
or you have to craft your bounce handling for your very specific local server
and any other possible servers you talk with. Even then you can have problems
dealing with bounces reliably.
Inside this description is an important concept to understand:
Lamson does NOT process bounces from the remote MTA or the remote user in
any way. Lamson process bounces from the localSMTP server.
This is important because if you try to use the bounce message as if it comes
from the remote user, then you’ll accidentally put your local server into a
state that prevents you from processing future bounce messages.
If you did not get that, reread this whole section again until moving forward.
“Hard” vs. “Soft”
Another complexity in dealing with bounces is the concept of “Hard” vs. “Soft”
bounces. My opinion is that the distinction is fairly retarded since it is
almost entirely unreliable and has no meaning to someone trying to handle a
bounce.
In popular terminology the main difference is this:
Hard bounce means that person does not exist, so I can not sell him any more crap and need to remove him (maybe).
Soft bounce means that person still exists, but my marketing bullshit didn’t get through, I should try again 10 or 15 more times until he gets my important message about winning the lottery.
You may be laughing, or screaming various pedantic specifics, but the above two
distinctions are both how many email services look at bounces, and how most
malicious mail users view them.
For the mail services, leaking a hard or soft bounce is a security problem
because it tells a malicious sender if that address is a valid person, why the
sender was blocked, and how they can work around it. This is why many of the
error messages you get back are vague and mostly lies. The major email
services just don’t want to give you any information that you can use to
circumvent their anti-spam measures.
The services that have the strictest anti-spam measures also have the most
nasty disgusting spam on the web pages displaying user’s mail. Yahoo is both
the worst for erroneously blocking email and for showing the worst most tricky
spam all over every square inch of their mail service.
How does this relate to your Lamson application? It basically says that you
should treat bounces as being basically soft bounces all the time, and then
tune your rules heuristically over time as you run into more and more.
VERP
The final topic to touch on before getting into how Lamson handles bounces is
one of VERP. Remember in the description of bounce handling above I said that
your localMTA sends the bounce message back to the envelope from not the
From in the headers (well, most ones will). Because of this you can have a
From in the envelope that is only replied to when there’s a bounce, and then
put the real From in the headers for normal processing.
This is effectively what Variable Envelope Return
Path does to
process bounce messages. Rather than attempt to parse the body of a bounce
message, VERP rewrite the From address so that when a bounce is returned, you
only have to process the returned address to see what the original was.
Lamson could potentially support this, but it has several problems for generic
bounce handling which meant that actually parsing bounce bodies was a better
option.
Using Lamson’s Bounce Handling
Using Lamson’s bounce handling is fairly simple. It involves the following
process:
Import a special decorator bounce_to from lamson.bounce
Create two (or one) handlers to deal with bounces.
Place the decorator on any START entry points to your handlers that can receive bounces, pointing them at your handlers.
Handle the bounces in your handlers, making sure to return back to the START state for the local MTA (remember, the local MTA is a person for bounce handling).
Here’s some simple code that does exactly this by just ignoring bounces from
the myinboxisnota.tv example:
This example is stripped down from the real code so you can see what’s going
on. If we walk through this you can see what happens:
First we import the BOUNCES variable from config.settings which is just the queue we want to dump bounces into.
We then create a special handler named IGNORE_BOUNCE that accepts a message to anything in its route and just puts the message in the BOUNCES queue.
This IGNORE_BOUNCE handler then immediately returns START so we go back to the START state.
On the START state you’ll see that we have our bounce_to declaration that points for hard and soft bounces at IGNORE_BOUNCE.
This decorator wraps your handler in a little check that, if the message is a bounce, your START state won’t get called, and instead your IGNORE_BOUNCE state will.
That is pretty much all you need to deal with to re-route bounces somewhere else.
You can also redirect them to a totally different handler, which is exactly
what the librelist.com example does.
How It Works
How does Lamson figure out that something is a bounce? What Lamson does is it assumes bounces will have some or all of these
headers:
The problem traditionally with parsing a bounce message was that, while you
need to find all of these headers, there was no real standard as to how the
messages in the bounce message are structured. From my
postfix server the bounce message is a sequence of
about 6 nested attachments containing other messages, and sometimes the nesting
goes three deep.
Rather than rely on this structure (which changes all the time) or that these
headers are always present (they aren’t), Lamson takes a probabilistic approach
based on the number of headers and properly formatted values it finds in all
nested attachments. The process goes something like this:
Traverse all the possible nested attachments.
Try to find each header in the attachment. If it’s found add a point.
If the header is found, use the regex associated with it (above) to try to match the value.
If the value matches, then keep the regex captures for later. Add another point.
For each header found, and any regex captures that matched the bodies, put them into an internal dict for analyzing the bounce information.
Finally, calculate a probability score that is the total number of BOUNCE_MATCHERS * 2.0 / points.
In general, if a message is found that has a 0.3 or higher bounce probability
then it is considered a bounce message and you can look at it. The bounce_to
decorator has a threshold you can adjust if you want to be more or less strict.
The final result of this processing (which is actually very fast) is that any
calls to MailRequest.is_bounce will either return True or False, and then
after you call is_bounce you can access the MailRequest.bounce attribute to
analyze the information. That information is captured and cooked into a
BounceAnalyzer
object.
What It Looks Like To Receive One
It’s also instructive to see what it looks like when Lamson processes a bounce
message. Here’s the librelist.com server processing
a bounce message:
2009-08-21 13:43:47,223 - root - DEBUG - Pulled message with key:
'1250876622.V8e00I219de0M128371' off
2009-08-21 13:43:47,231 - root - DEBUG - Message received from Peer:
'/var/mail/vhosts/librelist.com/delivery/', From: u'"SPAMMER"
', to To [u'lamson@librelist.com'].
2009-08-21 13:43:47,251 - routing - DEBUG - Matched u'lamson@librelist.com'
against START.
2009-08-21 13:43:47,332 - routing - DEBUG - Message to
set([u'lamson@librelist.com']) was handled by app.handlers.admin.START
2009-08-21 13:43:57,367 - root - DEBUG - Pulled message with key:
'1250876627.V8e00I219661M719350' off
2009-08-21 13:43:57,381 - root - DEBUG - Message received from Peer:
'/var/mail/vhosts/librelist.com/delivery/', From:
u'MAILER-DAEMON@librelist.com (Mail Delivery System)', to To
[u'lamson-confirm-74e2ca94b24a4be18da277f4666a6494@librelist.com'].
2009-08-21 13:43:57,410 - routing - DEBUG - Matched
u'lamson-confirm-74e2ca94b24a4be18da277f4666a6494@librelist.com' against START.
2009-08-21 13:43:57,431 - routing - DEBUG - Message to
set([u'lamson-confirm-74e2ca94b24a4be18da277f4666a6494@librelist.com']) was
handled by app.handlers.admin.START
These log messages show the following interaction:
SPAMMER@hotmail.com tried to spam the lamson@librelist.com mailing list.
They were required to subscribe, so Lamson sent them a confirmation mail.
That message bounced, so Postfix sent back a bounce message from MAILER-DAEMON@librelist.com to Lamson.
This message from MAILER-DAEMON is a bounce, so the librelist code handled it on the START state, NOT the CONFIRMING_SUBSCRIBE state.
Internally, librelist looked up the target user and just zapped them.
That shows how the bounce doesn’t come from SPAMMER@hotmail.com nor any server
at hotmail.com, but instead from MAILER-DAEMON@librelist.com. You could also
get messages from a remote MTA, but if they were found to be bounce messages
then that remote MTA would be treated like your own MTA.
Gettting Bounce Information From BounceAnalyzer
The BounceAnalyzer
does the work of figuring out additional useful information you can use to
determine what to do with the bounce. It looks at the final headers that are
scanned in the above process and pulls out important information everyone
needs. The list of information you can get is:
primary_status — The main status number that determines hard vs soft.
secondary_status — Advice status.
combined_status — the 2nd and 3rd number combined gives more detail.
remote_mta — The MTA that you sent mail to and aborted.
reporting_mta — The MTA that was sending the mail and has to report to you.
diagnostic_codes — Human readable codes usually with info from the provider.
action — Usually 'failed’, and turns out to be not too useful.
content_parts — All the attachments found as a hash keyed by the type.
original — The original message, if it’s found.
report — All report elements, as lamson.encoding.MailBase raw messages.
notification — Usually the detailed reason you bounced.
But, refer to the documentation for more accurate listings. An important
feature is that the status codes are parsed and converted into a standard list
available in lamson.bounce based on their numeric values. Rather than parse
the details given by the remote MTA, you just use the number codes to get a
human readable output.
The best way to see all that’s possible is to take a glance at the Lamson unit
test for the BounceAnalyzer:
Here you can see that you can figure out if the bounce is hard vs. soft, get a
human description, access status codes of various flavors, get the remote MTA’s
name, the reporting MTA (your local), who it was originally for
(final_recipient), and even access the raw bounce.headers if that’s not good
enough.
Augmenting The Matchers
Another advantage of this method of processing the bounces is that if your SMTP
server crafts something that hasn’t been handled, then you can augment the
matchers being used. Simply update the lamson.bounce.BOUNCE_MATCHERS dict
with your new ones and make sure to update lamson.bounce.BOUNCE_MAX to be 2
times that.
The status codes are also available in the same way. Refer to the source for
more information.
One tricky part of Lamson’s bounce handling is that it does assume a certain
structure for the BounceAnalyzer to get at the internal details. This
structure is the one used by Postfix, and it should be the same for other
servers. However, if you run into a structural difference, report the results
back so the handling can be improved.
A More Complete Example
Finally, the librelist.com example code has a much more
complete example of using bounces to disable users and shift their state.
Rather than describe it in detail here, I’ll simply point you at the source
releases so you can see it for yourself. Look in
examples/librelist/app/handlers/bounce.py to see how it all works.
In fact, studying how this is triggered from the rest of the librelist example
is a great way to learn how to use Lamson in an advanced fashion. Study well.
Conclusion
Lamson bounce handling is very advanced and can deal with a wide range of
scenarios. It should work with a wide range of bounce styles and other
servers, but feel free to report your own experiences and differences.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/bounce_detection.txt 0000644 0000765 0000024 00000043554 11243654672 025646 0 ustar zedshaw staff Title: Bounce Detection
Lamson supports intelligent bounce detection with its
"lamson.bounce":http://lamsonproject.org/docs/api/lamson.bounce-module.html
module. The bounce handling is based on a probability that, depending on found
headers, the message is a bounce. It then gives you a nice clean interface to
check who it's from, the originating SMTP server, the error messages, and any
human readable messages.
h1. How Bounces Actually Work
Figuring out how a bounce is actually handled is a bit difficult because the
majority of the information available is written by people who know very little
of SMTP server operations. When the average programmer thinks of handling a
bounce, she usually has one of these concepts in mind:
# The message could not be delivered, so the remote SMTP server sent back a bounce message.
# The message could not be delivered, so the local SMTP server sent back a bounce message.
# The message could not be delivered, so the recipient's email client sent back a bounce message.
# The message could not be delivered, so Lamson crafted a bounce message.
Obviously logically you can't have the recipient's email client send you a
bounce unless the user does something weird (and incredibly annoying). This
makes sense because the message was @not delivered@. How can the email client
send back a bounce if they don't receive the message.
bq. Yes, some clients do support sending bounces, but very few people use this
feature. If you do please talk about it as one more edge case to deal with.
Next Lamson can't craft the bounce messages for you because Lamson is simply
trying to deliver outgoing mail to a smart-host. Lamson does nothing but ask
the smart-host (your local SMTP server) to deliver, and then waits for a
response. That means Lamson is not sending you any kind of bounce unless you
write code to make it do that.
That leaves only two options for @who@ is really sending the bounce message:
the remote MTA or the local MTA. The truth is it's a little bit of both.
h2. Your Local MTA Is A Person
How a bounce works in practice involves two SMTP servers: your local
smart-host, and the remote server that it tries to connect to for delivery.
The message you actually get in lamson is from the @local@ server, and usually
an address @at@ that local server. It does not come from the remote server,
but inside your bounce will be a message and status indicators from the remote
server indicating why it bounced.
What happens is your local server attempts to deliver, and the remote server
rejects it for some reason. Now your local server is supposed to try again in
certain situations, but after a certain number of rejections or failures it
crafts a bounce message. That bounce message is then returned to your @lamson@
server based on the @envelope@ header of the message (more on that later).
Now, what's inside this message? Well, it's a oddly nested barely standardized
pile of random other messages. This is the frustration with bounce handling.
You pretty much either have to use a probability mechanism (like Lamson does)
or you have to craft your bounce handling for your very specific local server
and any other possible servers you talk with. Even then you can have problems
dealing with bounces reliably.
Inside this description is an important concept to understand:
bq. Lamson does NOT process bounces from the remote MTA or the remote user in
any way. Lamson process bounces from the @local@ SMTP server.
This is important because if you try to use the bounce message as if it comes
from the remote user, then you'll accidentally put your @local@ server into a
state that prevents you from processing future bounce messages.
If you did not get that, reread this whole section again until moving forward.
h2. "Hard" vs. "Soft"
Another complexity in dealing with bounces is the concept of "Hard" vs. "Soft"
bounces. My opinion is that the distinction is fairly retarded since it is
almost entirely unreliable and has no meaning to someone trying to handle a
bounce.
In popular terminology the main difference is this:
* Hard bounce means that person does not exist, so I can not sell him any more crap and need to remove him (maybe).
* Soft bounce means that person still exists, but my marketing bullshit didn't get through, I should try again 10 or 15 more times until he gets my important message about winning the lottery.
You may be laughing, or screaming various pedantic specifics, but the above two
distinctions are both how many email services look at bounces, and how most
malicious mail users view them.
For the mail services, leaking a hard or soft bounce is a security problem
because it tells a malicious sender if that address is a valid person, why the
sender was blocked, and how they can work around it. This is why many of the
error messages you get back are vague and mostly lies. The major email
services just don't want to give you any information that you can use to
circumvent their anti-spam measures.
bq. The services that have the strictest anti-spam measures also have the most
nasty disgusting spam on the web pages displaying user's mail. Yahoo is both
the worst for erroneously blocking email and for showing the worst most tricky
spam all over every square inch of their mail service.
How does this relate to your Lamson application? It basically says that you
should treat bounces as being basically soft bounces all the time, and then
tune your rules heuristically over time as you run into more and more.
h2. VERP
The final topic to touch on before getting into how Lamson handles bounces is
one of VERP. Remember in the description of bounce handling above I said that
your @local@ MTA sends the bounce message back to the @envelope from@ not the
From in the headers (well, most ones will). Because of this you can have a
From in the envelope that is only replied to when there's a bounce, and then
put the real From in the headers for normal processing.
This is effectively what "Variable Envelope Return
Path":http://en.wikipedia.org/wiki/Variable_envelope_return_path does to
process bounce messages. Rather than attempt to parse the body of a bounce
message, VERP rewrite the From address so that when a bounce is returned, you
only have to process the returned address to see what the original was.
Lamson could potentially support this, but it has several problems for generic
bounce handling which meant that actually parsing bounce bodies was a better
option.
h2. Using Lamson's Bounce Handling
Using Lamson's bounce handling is fairly simple. It involves the following
process:
# Import a special decorator @bounce_to@ from "lamson.bounce":http://lamsonproject.org/docs/api/lamson.bounce-module.html
# Create two (or one) handlers to deal with bounces.
# Place the decorator on any @START@ entry points to your handlers that can receive bounces, pointing them at your handlers.
# Handle the bounces in your handlers, making sure to return back to the @START@ state for the local MTA (remember, the local MTA is a person for bounce handling).
Here's some simple code that does exactly this by just ignoring bounces from
the "myinboxisnota.tv":http://myinboxisnota.tv/ example:
This example is stripped down from the real code so you can see what's going
on. If we walk through this you can see what happens:
# First we import the @BOUNCES@ variable from @config.settings@ which is just the queue we want to dump bounces into.
# We then create a special handler named @IGNORE_BOUNCE@ that accepts a message to anything in its @route@ and just puts the message in the @BOUNCES@ queue.
# This @IGNORE_BOUNCE@ handler then immediately returns @START@ so we go back to the START state.
# On the @START@ state you'll see that we have our @bounce_to@ declaration that points for @hard@ and @soft@ bounces at @IGNORE_BOUNCE@.
# This decorator wraps your handler in a little check that, if the message is a bounce, your @START@ state won't get called, and instead your @IGNORE_BOUNCE@ state will.
That is pretty much all you need to deal with to re-route bounces somewhere else.
You can also redirect them to a totally different handler, which is exactly
what the "librelist.com":http://librelist.com/ example does.
h2. How It Works
How does Lamson figure out that something is a bounce? What Lamson does is it assumes bounces will have some or all of these
headers:
The problem traditionally with parsing a bounce message was that, while you
need to find all of these headers, there was no real standard as to how the
messages in the bounce message are structured. From my
"postfix":http://www.postfix.org/ server the bounce message is a sequence of
about 6 nested attachments containing other messages, and sometimes the nesting
goes three deep.
Rather than rely on this structure (which changes all the time) or that these
headers are always present (they aren't), Lamson takes a probabilistic approach
based on the number of headers and properly formatted values it finds in @all@
nested attachments. The process goes something like this:
# Traverse all the possible nested attachments.
# Try to find each header in the attachment. If it's found add a point.
# If the header is found, use the regex associated with it (above) to try to match the value.
## If the value matches, then keep the regex captures for later. Add another point.
# For each header found, and any regex captures that matched the bodies, put them into an internal dict for analyzing the bounce information.
# Finally, calculate a probability score that is the total number of BOUNCE_MATCHERS * 2.0 / points.
In general, if a message is found that has a 0.3 or higher bounce probability
then it is considered a bounce message and you can look at it. The @bounce_to@
decorator has a threshold you can adjust if you want to be more or less strict.
The final result of this processing (which is actually very fast) is that any
calls to @MailRequest.is_bounce@ will either return True or False, and then
after you call is_bounce you can access the @MailRequest.bounce@ attribute to
analyze the information. That information is captured and cooked into a
"BounceAnalyzer":http://lamsonproject.org/docs/api/lamson.bounce.BounceAnalyzer-class.html
object.
h2. What It Looks Like To Receive One
It's also instructive to see what it looks like when Lamson processes a bounce
message. Here's the "librelist.com":http://librelist.com/ server processing
a bounce message:
2009-08-21 13:43:47,223 - root - DEBUG - Pulled message with key:
'1250876622.V8e00I219de0M128371' off
2009-08-21 13:43:47,231 - root - DEBUG - Message received from Peer:
'/var/mail/vhosts/librelist.com/delivery/', From: u'"SPAMMER"
', to To [u'lamson@librelist.com'].
2009-08-21 13:43:47,251 - routing - DEBUG - Matched u'lamson@librelist.com'
against START.
2009-08-21 13:43:47,332 - routing - DEBUG - Message to
set([u'lamson@librelist.com']) was handled by app.handlers.admin.START
2009-08-21 13:43:57,367 - root - DEBUG - Pulled message with key:
'1250876627.V8e00I219661M719350' off
2009-08-21 13:43:57,381 - root - DEBUG - Message received from Peer:
'/var/mail/vhosts/librelist.com/delivery/', From:
u'MAILER-DAEMON@librelist.com (Mail Delivery System)', to To
[u'lamson-confirm-74e2ca94b24a4be18da277f4666a6494@librelist.com'].
2009-08-21 13:43:57,410 - routing - DEBUG - Matched
u'lamson-confirm-74e2ca94b24a4be18da277f4666a6494@librelist.com' against START.
2009-08-21 13:43:57,431 - routing - DEBUG - Message to
set([u'lamson-confirm-74e2ca94b24a4be18da277f4666a6494@librelist.com']) was
handled by app.handlers.admin.START
These log messages show the following interaction:
# SPAMMER@hotmail.com tried to spam the lamson@librelist.com mailing list.
# They were required to subscribe, so Lamson sent them a confirmation mail.
# That message bounced, so Postfix sent back a bounce message from MAILER-DAEMON@librelist.com to Lamson.
# This message from MAILER-DAEMON is a bounce, so the librelist code handled it on the START state, NOT the CONFIRMING_SUBSCRIBE state.
# Internally, librelist looked up the target user and just zapped them.
That shows how the bounce doesn't come from SPAMMER@hotmail.com nor any server
at hotmail.com, but instead from MAILER-DAEMON@librelist.com. You could also
get messages from a remote MTA, but if they were found to be bounce messages
then that remote MTA would be treated like your own MTA.
h2. Gettting Bounce Information From BounceAnalyzer
The "BounceAnalyzer":http://lamsonproject.org/docs/api/lamson.bounce.BounceAnalyzer-class.html
does the work of figuring out additional useful information you can use to
determine what to do with the bounce. It looks at the final headers that are
scanned in the above process and pulls out important information everyone
needs. The list of information you can get is:
* primary_status -- The main status number that determines hard vs soft.
* secondary_status -- Advice status.
* combined_status -- the 2nd and 3rd number combined gives more detail.
* remote_mta -- The MTA that you sent mail to and aborted.
* reporting_mta -- The MTA that was sending the mail and has to report to you.
* diagnostic_codes -- Human readable codes usually with info from the provider.
* action -- Usually 'failed', and turns out to be not too useful.
* content_parts -- All the attachments found as a hash keyed by the type.
* original -- The original message, if it's found.
* report -- All report elements, as lamson.encoding.MailBase raw messages.
* notification -- Usually the detailed reason you bounced.
But, refer to the documentation for more accurate listings. An important
feature is that the status codes are parsed and converted into a standard list
available in @lamson.bounce@ based on their numeric values. Rather than parse
the details given by the remote MTA, you just use the number codes to get a
human readable output.
The best way to see all that's possible is to take a glance at the Lamson unit
test for the BounceAnalyzer:
Here you can see that you can figure out if the bounce is hard vs. soft, get a
human description, access status codes of various flavors, get the remote MTA's
name, the reporting MTA (your local), who it was originally for
(final_recipient), and even access the raw @bounce.headers@ if that's not good
enough.
h2. Augmenting The Matchers
Another advantage of this method of processing the bounces is that if your SMTP
server crafts something that hasn't been handled, then you can augment the
matchers being used. Simply update the @lamson.bounce.BOUNCE_MATCHERS@ dict
with your new ones and make sure to update @lamson.bounce.BOUNCE_MAX@ to be 2
times that.
The status codes are also available in the same way. Refer to the source for
more information.
One tricky part of Lamson's bounce handling is that it does assume a certain
structure for the BounceAnalyzer to get at the internal details. This
structure is the one used by Postfix, and it should be the same for other
servers. However, if you run into a structural difference, report the results
back so the handling can be improved.
h2. A More Complete Example
Finally, the "librelist.com":http://librelist.com/ example code has a much more
complete example of using bounces to disable users and shift their state.
Rather than describe it in detail here, I'll simply point you at the "source
releases":/releases/ so you can see it for yourself. Look in
@examples/librelist/app/handlers/bounce.py@ to see how it all works.
In fact, studying how this is triggered from the rest of the librelist example
is a great way to learn how to use Lamson in an advanced fashion. Study well.
h2. Conclusion
Lamson bounce handling is very advanced and can deal with a wide range of
scenarios. It should work with a wide range of bounce styles and other
servers, but feel free to report your own experiences and differences.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/confirmations.html 0000644 0000765 0000024 00000045251 11310760166 025320 0 ustar zedshaw staff
LamsonProject: Confirmations
You never know who’s going to send an email to your Lamson application. It may
be a real person, or a spam bot. Currently most of the spam bots aren’t very
intelligent (probably because they haven’t discovered Lamson yet) so a simple
easy way to weed out the randomness is to confirm people at important “choke
points” in your application.
Lamson provides a simple API in
lamson.confirm
to help you send out and verify confirmation emails. You provide a storage
class for keeping track of who you’re expecting to confirm, a template for your
message to send, and it does the rest. It’s even advanced enough to keep the
original email around so you can resend it after the confirmation.
The General Theory
The confirmation API assumes a few things about how you’re doing your
confirmations:
You want them to reply to an email to confirm.
The address they reply to confirm has a “target” to differentiate it from other parts of your application that needs confirmation.
The address they reply to also has a hex formatted randomly generated UUID for security, and your storage can handle that length.
You want to store the message they sent so you can get it back after they confirm.
With those fairly reasonable assumptions you only need to then setup the
confirmation API in your config/settings.py file and use it in your handlers,
preferrably at your START state and anywhere else that you need
to validate the user before they do something destructive.
Simplest Usage
The most basic usage of the
lamson.confirmAPI is to put it in your config/settings.py and then access it in your
handlers. You configure it like this:
from lamson import confirm
...
CONFIRM_STORAGE=confirm.ConfirmationStorage()
CONFIRM = confirm.ConfirmationEngine('run/pending')
This puts a variable in your settings.py that you can access from your handlers
to craft and verify the confirmation messages.
By default uses a simple in-memory dict to store the confirmations. That’s
fine for testing and an initial deploy, but you’ll probably want to switch to a
permanent form of storage for later. Further on in this document you’ll see
how.
Next, you need to use it in your START state, and then in a CONFIRMING
state. Here’s a simple example:
First we import the CONFIRM variable so we can use it.
In our START handler (which is accepts start@(host)) we use the API to send out the confirmation message they should reply-to. Notice how we give a “start” target as the second argument, this is important.
Then we transition to CONFIRMING and wait for them to reply to that message.
The user then replies to the message we sent, so we handle the CONFIRMING state. Notice that we are handling “start-confirm-(id_number)” as the initial message, with “start” being the target (2nd parameter) from our above CONFIRM.send call.
In CONFIRMING we use the CONFRIM.verify method to validate that it’s from the right person, to the right target (“start”) and that they got the secret (id_number) right.
Finally, if it’s right we send them a welcome message, and if it’s not we just ignore the message.
An alternative to ignoring the failed confirmation from them is to cancel it
and go back to the START state for them. The danger with that method though is
a spam bot will get into a loop where you are sending them constant
confirmation messages in a loop between START and CONFIRMING. It’s best to
drop it, and maybe provide a “cancel” mechanism.
Using Shelf Storage
Other than a few other methods, there’s only a need to change the storage. The
simplest change is to provide a dict-like interface to the
lamson.confirm.ConfirmationEngine
to store. Easiest available is the Python
shelf module which gives a dict
interface to various key/value storage backends.
To use one, just change your code in config/settings.py to be like this:
All you do is create a
lamson.confirm.ConfirmationStorage
and give the db= parameter a dict it can use. Everything else will be the
same.
There might be thread issues with this, and it will definitely fail if you use mulitple processes. See the next
section on using a Django Model.
Using A Django ORM Model
In the librelist.com example code you’ll find that it
stores the confirmations in the Django model in the webapp/librelist
directory. This is actually easily setup, so first read Hooking Into
Django to learn how to access a Django ORM.
After that, you write a simple version of ConfirmationStorage that would look
something like this:
This is from Librelist, so you see we just import the Confirmation model and
then wrap it with the get, delete, set, and clear methods that
ConfirmationEngine needs to run.
For completeness, here’s what the Django Confirmation model looks like:
Final step is to configure it in your config/settings.py thusly:
from lamson import confirm
...
from app.model.confirmation import DjangoConfirmStorage
CONFIRM = confirm.ConfirmationEngine('run/pending', DjangoConfirmStorage())
That’s all there is to it. This is actually a nice setup because you can use
the Django Admin to manage it during your first deployments.
Other ORM
For other ORM systems simply use the same pattern as the Django example above.
You just create a similar model, wrap it with your own version of
ConfirmationStorage and plug it into the ConfirmationEngine you use.
Targets
The only other thing to understand is why the API has a “target” parameter.
Let’s look at the call to CONFIRM.send again:
The “start” string as the second parameter acts as a the target. It says that
this user needs to confirm for the “start” target when they do their reply.
That’s why the route on CONFIRMING is then like this:
This way, the user could have multiple simultaneous confirmations going for
different lists and they won’t step on eachother.
Without this differentiator, you’d have to either restrict users to just one
confirmation at a time, or you’d end up getting the data all confused.
Confirming Off A Web Link
If you want people to go to a web link instead of simply replying, then you have to do the
following:
Either write your own version, or subclass ConfirmationEngine so that it uses an address they can’t reply to.
Make sure you use an ORM that can access your database and store both the confirmation info, and each users’s state.
When the user hits the link you give them and does whatever you need, use the web framework’s ORM to validate their confirmation.
Once your web framework has validated their confirmation, then change their state in Lamson’s state using your web framework ORM out of CONFIRMING and into the next state.
Assuming you’re doing this all with Python it should be fairly trivial.
Confirm Only By Web Is Bad
I would advise against this method though, since it doesn’t really confirm that
the email address you received worked. One of the purposes of doing a
confirmation email exchange is to make sure that this person can both send
and receive. If you have to point them at your web site, consider having
this process instead:
Their first interaction with your service sends out an email that sends them to a web page, and transitions them to a CONFIRMING state but do not send them a confirmation reply address from Lamson. You’ll actually “delay” this until they fill out your web forms.
In your web framework, you have them fill out forms and such, and then send them the real confirmation message using Lamson. Since the Confirmation API is Python you could do this directly in any Python web framework. You’re basically moving the call to CONFRIM.send from your START handler into the web framework.
Then your web framework will have sent them a real confirmation email, not just a link, so when they reply, continue with the usual Lamson confirmation process described above.
Doing it this way ends up being a good balance between too many clicks and replies, but too few to confirm that the end user can actually reply
to email you send them.
The Pending Queue
You should also notice in the above examples that the original message is
stored in a “pending queue” and then given back to you later. This is handy
for either finishing their original request without further intervention, or
inspecting what they original wanted to do. In the original Librelist code I
would take their first message, confirm them, and then pull it out of the
pending queue to send it on. This turned out to not work because socially
people “subscribe” with a garbage first message, but technically it worked
great.
You may want to periodically go through this queue and purge any messages that aren’t found in the ConfirmationsStorage. Probably with a simple Python script and a cronjob.
Conclusion
The Lamson confirmation API encapsulates a pattern for confirming potential
users. Feel free to suggest improvements to the API if you find further
patterns that are needed.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/confirmations.txt 0000644 0000765 0000024 00000030576 11310760160 025171 0 ustar zedshaw staff Title: Confirmations
You never know who's going to send an email to your Lamson application. It may
be a real person, or a spam bot. Currently most of the spam bots aren't very
intelligent (probably because they haven't discovered Lamson yet) so a simple
easy way to weed out the randomness is to confirm people at important "choke
points" in your application.
Lamson provides a simple API in
"lamson.confirm":http://lamsonproject.org/docs/api/lamson.confirm-module.html
to help you send out and verify confirmation emails. You provide a storage
class for keeping track of who you're expecting to confirm, a template for your
message to send, and it does the rest. It's even advanced enough to keep the
original email around so you can resend it after the confirmation.
h2. The General Theory
The confirmation API assumes a few things about how you're doing your
confirmations:
# You want them to reply to an email to confirm.
# The address they reply to confirm has a "target" to differentiate it from other parts of your application that needs confirmation.
# The address they reply to also has a hex formatted randomly generated UUID for security, and your storage can handle that length.
# You want to store the message they sent so you can get it back after they confirm.
With those fairly reasonable assumptions you only need to then setup the
confirmation API in your @config/settings.py@ file and use it in your handlers,
preferrably at your @START@ state and anywhere else that you need
to validate the user before they do something destructive.
h2. Simplest Usage
The most basic usage of the
"lamson.confirm":http://lamsonproject.org/docs/api/lamson.confirm-module.html
API is to put it in your @config/settings.py@ and then access it in your
handlers. You configure it like this:
from lamson import confirm
...
CONFIRM_STORAGE=confirm.ConfirmationStorage()
CONFIRM = confirm.ConfirmationEngine('run/pending')
This puts a variable in your settings.py that you can access from your handlers
to craft and verify the confirmation messages.
bq. By default uses a simple in-memory dict to store the confirmations. That's
fine for testing and an initial deploy, but you'll probably want to switch to a
permanent form of storage for later. Further on in this document you'll see
how.
Next, you need to use it in your @START@ state, and then in a @CONFIRMING@
state. Here's a simple example:
Here's how the above code works:
# First we import the CONFIRM variable so we can use it.
# In our @START@ handler (which is accepts start@(host)) we use the API to send out the confirmation message they should reply-to. Notice how we give a "start" target as the second argument, this is important.
# Then we transition to CONFIRMING and wait for them to reply to that message.
# The user then replies to the message we sent, so we handle the CONFIRMING state. Notice that we are handling "start-confirm-(id_number)" as the initial message, with "start" being the target (2nd parameter) from our above @CONFIRM.send@ call.
# In CONFIRMING we use the @CONFRIM.verify@ method to validate that it's from the right person, to the right target ("start") and that they got the secret (id_number) right.
# Finally, if it's right we send them a welcome message, and if it's not we just ignore the message.
An alternative to ignoring the failed confirmation from them is to cancel it
and go back to the START state for them. The danger with that method though is
a spam bot will get into a loop where you are sending them constant
confirmation messages in a loop between START and CONFIRMING. It's best to
drop it, and maybe provide a "cancel" mechanism.
h2. Using Shelf Storage
Other than a few other methods, there's only a need to change the storage. The
simplest change is to provide a dict-like interface to the
"lamson.confirm.ConfirmationEngine":http://lamsonproject.org/docs/api/lamson.confirm.ConfirmationEngine-class.html
to store. Easiest available is the Python
"shelf":http://docs.python.org/library/shelve.html module which gives a dict
interface to various key/value storage backends.
To use one, just change your code in @config/settings.py@ to be like this:
All you do is create a
"lamson.confirm.ConfirmationStorage":http://lamsonproject.org/docs/api/lamson.confirm.ConfirmationStorage-class.html
and give the @db=@ parameter a dict it can use. Everything else will be the
same.
bq. There might be thread issues with this, and it will definitely fail if you use mulitple processes. See the next
section on using a Django Model.
h2. Using A Django ORM Model
In the "librelist.com":http://librelist.com/ example code you'll find that it
stores the confirmations in the Django model in the @webapp/librelist@
directory. This is actually easily setup, so first read "Hooking Into
Django":/docs/hooking_into_django.html to learn how to access a Django ORM.
After that, you write a simple version of @ConfirmationStorage@ that would look
something like this:
This is from Librelist, so you see we just import the @Confirmation@ model and
then wrap it with the @get@, @delete@, @set@, and @clear@ methods that
@ConfirmationEngine@ needs to run.
For completeness, here's what the Django @Confirmation@ model looks like:
Final step is to configure it in your @config/settings.py@ thusly:
from lamson import confirm
...
from app.model.confirmation import DjangoConfirmStorage
CONFIRM = confirm.ConfirmationEngine('run/pending', DjangoConfirmStorage())
That's all there is to it. This is actually a nice setup because you can use
the Django Admin to manage it during your first deployments.
h2. Other ORM
For other ORM systems simply use the same pattern as the Django example above.
You just create a similar model, wrap it with your own version of
@ConfirmationStorage@ and plug it into the @ConfirmationEngine@ you use.
h2. Targets
The only other thing to understand is why the API has a "target" parameter.
Let's look at the call to CONFIRM.send again:
The "start" string as the second parameter acts as a the target. It says that
this user needs to confirm for the "start" target when they do their reply.
That's why the @route@ on @CONFIRMING@ is then like this:
This way, the user could have multiple simultaneous confirmations going for
different lists and they won't step on eachother.
Without this differentiator, you'd have to either restrict users to just one
confirmation at a time, or you'd end up getting the data all confused.
h2. Confirming Off A Web Link
If you want people to go to a web link instead of simply replying, then you have to do the
following:
# Either write your own version, or subclass @ConfirmationEngine@ so that it uses an address they can't reply to.
# Make sure you use an ORM that can access your database and store both the confirmation info, and each users's state.
# When the user hits the link you give them and does whatever you need, use the web framework's ORM to validate their confirmation.
# Once your web framework has validated their confirmation, then change their state *in Lamson's state* using *your web framework ORM* out of CONFIRMING and into the next state.
Assuming you're doing this all with Python it should be fairly trivial.
h2. Confirm Only By Web Is Bad
I would advise against this method though, since it doesn't really confirm that
the email address you received worked. One of the purposes of doing a
confirmation email exchange is to make sure that this person can both *send*
and *receive*. If you have to point them at your web site, consider having
this process instead:
# Their first interaction with your service sends out an email that sends them to a web page, and transitions them to a @CONFIRMING@ state *but do not send them a confirmation reply address from Lamson.* You'll actually "delay" this until they fill out your web forms.
# In your web framework, you have them fill out forms and such, and then send them the *real* confirmation message using Lamson. Since the Confirmation API is Python you could do this directly in any Python web framework. You're basically moving the call to @CONFRIM.send@ from your @START@ handler into the web framework.
# Then your web framework will have sent them a real confirmation email, not just a link, so when they reply, continue with the usual Lamson confirmation process described above.
Doing it this way ends up being a good balance between too many clicks and replies, but too few to confirm that the end user can actually reply
to email you send them.
h2. The Pending Queue
You should also notice in the above examples that the original message is
stored in a "pending queue" and then given back to you later. This is handy
for either finishing their original request without further intervention, or
inspecting what they original wanted to do. In the original Librelist code I
would take their first message, confirm them, and then pull it out of the
pending queue to send it on. This turned out to not work because socially
people "subscribe" with a garbage first message, but technically it worked
great.
bq. You may want to periodically go through this queue and purge any messages that aren't found in the ConfirmationsStorage. Probably with a simple Python script and a cronjob.
h2. Conclusion
The Lamson confirmation API encapsulates a pattern for confirming potential
users. Feel free to suggest improvements to the API if you find further
patterns that are needed.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/deferred_processing_to_queues.html 0000644 0000765 0000024 00000031153 11242707507 030552 0 ustar zedshaw staff
LamsonProject: Deferred Processing To Queues
As of the 0.9.2 release there is preliminary support for deferring handling of
a mail message to a queue for another process to deal with in a separate
handler. This support is rough at this time, but still useful and not too
difficult to configure. As the feature gets more use it will improve and
hopefully turn into a generic “defer to queue” system in Lamson.
What is meant by “defer to queue” is simply that you take messages your state
function receives and you dump them into a maildir queue. You then have
another separate process read from this queue and do the real work.
Potentially you could have many processes deal with this work, and they could
even be on multiple computers.
A More Concrete Example
Imagine that you have a blog posting system and you want to update a big “front
page index” that shows recent posts by your users. However, you don’t want to
generate this index on every blog post users make, since that could involve
expensive computation and hold up other threads that need to deal with more
urgent email.
The solution is to do the minimum quick processing you can in your POSTING
state function, and then use the
lamson.queue.Queue
to queue up messages meant for “front page indexing”. Here’s how that code
might go:
@route("(post_name)@osb\\.(host)")
def POSTING(message, post_name=None, host=None):
# do the regular posting to blog thing
name, address = parseaddr(message['from'])
post.post(post_name, address, message)
msg = view.respond('page_ready.msg', locals())
relay.deliver(msg)
# drop the message off into the 'posts' queue for later
index_q = queue.Queue("run/posts")
index_q.push(message)
return POSTING
You can see that you just drop it into the queue with push(message) and it’s
done. What you don’t see is how this then gets picked up by another process to
actually do somehing with this email.
Configuring A config/queue.py
In Lamson you are given control over how your software boots, which gives you
the ability to configure extra services how you need. By default the lamson
gen command just outputs a basic config/boot.py and config/testing.py file
so you can get working, and these will work for most development purposes.
In this tutorial you get to write a new boot configuration and tell Lamson how
to use it. We’ll be copying the original boot file over first:
$ cp config/boot.py config/queue.py
Next you want to edit this file so that instead of running an
SMTPReceiver
it will use a
QueueReceiver
configured to pull out of the run/posts queue you are using in your POSTING
handler.
...
# where to listen for incoming messages
settings.receiver = QueueReceiver(settings.queue_config['queue'],
settings.queue_config['sleep'])
settings.database = configure_database(settings.database_config, also_create=False)
Router.defaults(**settings.router_defaults)
# NOTE: this is using a different handlers variable in settings
Router.load(settings.queue_handlers)
Router.RELOAD=True
...
I’ve removed the code above the … and below it since it’s the same in the two
files. Notice that you have a QueueReceiver now, and that you are telling
the
Router
that it will use settings.queue_handlers for the list of handlers to load and
run.
You now add these two lines to your config/settings.py:
...
# this is for when you run the config.queue boot
queue_config = {'queue': 'run/posts', 'sleep': 10}
queue_handlers = ['app.handlers.index']
The queue_config variable is read by the config/queue.py file for the
QueueReceiver and the queue_handlers is fed to the Router as described
above.
Writing The Index Handler
You now have to write a new handler that is in app/handlers/index.py so that
this config.queue boot setup will load it and run it whenever a message hits
the run/queue. Here’s the code:
from lamson import queue
from lamson.routing import route, stateless
import logging
@route("(post_name)@osb\\.(host)")
@stateless
def START(message, post_name=None, host=None):
logging.debug("Got message from %s", message['from'])
This simple demonstration will just log what messages it receives so you can
make sure it is working.
There are two points to notice about this handler. First, it is marked
stateless because it will run independent of the regular Lamson server, and
you don’t want its parallel operations to interfere with your normal server’s
state operations. Second, it uses a Router.defaults named post_name that
you would add to your config.settings.router_defaults.
Once you have all this slightly complicated setup done you are ready to test
it.
Also note that the examples in the source releases have code
that does a deferred queue similar to this. Go look there for more code to
steal.
Running Your Queue Receiver
Run your logger and lamson server like normal:
$ lamson log
$ lamson start
Next, go look in your logs and make sure it works by running your unit
tests:
$ nosetests
................
----------------------------------------------------------------------
Ran 16 tests in 1.346s
OK
Your logs should look normal, but now you should see some files in the
run/posts/new directory:
$ ls run/posts/new/
1244080328.M408474P3147Q4.mycomputer.local
That’s the results of your POSTING handler putting the messages it receives
into your run/posts maildir queue.
If you’re running the code given above then you should see this in the
logs/lamson.log file:
...
DEBUG:root:Sleeping for 10 seconds...
DEBUG:root:Pulled message with key:
'1244080328.M408474P3147Q4.zed-shaws-macbook.local' off
DEBUG:root:Message received from Peer: 'run/posts', From:
'sender-1244080328.22@sender.com', to To
['test.blog.1244080328@osb.test.com'].
DEBUG:root:Got message from sender-1244080328.22@sender.com
DEBUG:root:Message to test.blog.1244080328@osb.test.com was handled by
app.handlers.index.START
Which means your queue receiver is running. You could in theory run as many
of these as you wanted, as long as their handlers are stateless.
When you’re done you can stop the whole setup with the following command:
$ lamson stop -ALL run
Stopping processes with the following PID files:
['run/log.pid', 'run/queue.pid', 'run/smtp.pid']
Attempting to stop lamson at pid 3092
Attempting to stop lamson at pid 3157
Attempting to stop lamson at pid 3096
Further Advanced Usage
This configuration is debatable whether it is very usable or not, but it works
and will improve as the project continues. To give you some ideas of what you
can do with it:
Defer activity to other machines or processes.
Receive messages from other mail systems that know maildir.
Deliver messages to other maildir aware systems.
Process messages from a web application, and possibly even generic work.
It might also be possible to actually make your state functions transition to
the queue handler states by simply having the function return the
module.FUNCTION that should be next. Take care with this though as it means
your end user’s actions are effectively blocked for that event until the next
run of the queue receiver.
Call For Suggestions
Feel free to offer suggestions in improving this setup (or even better code).
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/deferred_processing_to_queues.txt 0000644 0000765 0000024 00000017601 11242707504 030424 0 ustar zedshaw staff Title: Deferred Processing To Queues
As of the 0.9.2 release there is preliminary support for deferring handling of
a mail message to a queue for another process to deal with in a separate
handler. This support is rough at this time, but still useful and not too
difficult to configure. As the feature gets more use it will improve and
hopefully turn into a generic "defer to queue" system in Lamson.
What is meant by "defer to queue" is simply that you take messages your state
function receives and you dump them into a maildir queue. You then have
another separate process read from this queue and do the real work.
Potentially you could have many processes deal with this work, and they could
even be on multiple computers.
h2. A More Concrete Example
Imagine that you have a blog posting system and you want to update a big "front
page index" that shows recent posts by your users. However, you don't want to
generate this index on *every* blog post users make, since that could involve
expensive computation and hold up other threads that need to deal with more
urgent email.
The solution is to do the minimum quick processing you can in your POSTING
state function, and then use the
"lamson.queue.Queue":http://lamsonproject.org/docs/api/lamson.queue.Queue-class.html
to queue up messages meant for "front page indexing". Here's how that code
might go:
@route("(post_name)@osb\\.(host)")
def POSTING(message, post_name=None, host=None):
# do the regular posting to blog thing
name, address = parseaddr(message['from'])
post.post(post_name, address, message)
msg = view.respond('page_ready.msg', locals())
relay.deliver(msg)
# drop the message off into the 'posts' queue for later
index_q = queue.Queue("run/posts")
index_q.push(message)
return POSTING
You can see that you just drop it into the queue with @push(message)@ and it's
done. What you don't see is how this then gets picked up by another process to
actually do somehing with this email.
h2. Configuring A config/queue.py
In Lamson you are given control over how your software boots, which gives you
the ability to configure extra services how you need. By default the @lamson
gen@ command just outputs a basic @config/boot.py@ and @config/testing.py@ file
so you can get working, and these will work for most development purposes.
In this tutorial you get to write a new boot configuration and tell Lamson how
to use it. We'll be copying the original boot file over first:
$ cp config/boot.py config/queue.py
Next you want to edit this file so that instead of running an
"SMTPReceiver":http://lamsonproject.org/docs/api/lamson.server.SMTPReceiver-class.html
it will use a
"QueueReceiver":http://lamsonproject.org/docs/api/lamson.server.QueueReceiver-class.html
configured to pull out of the @run/posts@ queue you are using in your @POSTING@
handler.
...
# where to listen for incoming messages
settings.receiver = QueueReceiver(settings.queue_config['queue'],
settings.queue_config['sleep'])
settings.database = configure_database(settings.database_config, also_create=False)
Router.defaults(**settings.router_defaults)
# NOTE: this is using a different handlers variable in settings
Router.load(settings.queue_handlers)
Router.RELOAD=True
...
I've removed the code above the ... and below it since it's the same in the two
files. Notice that you have a @QueueReceiver@ now, and that you are telling
the
"Router":http://lamsonproject.org/docs/api/lamson.routing.RoutingBase-class.html
that it will use @settings.queue_handlers@ for the list of handlers to load and
run.
You now add these two lines to your @config/settings.py@:
...
# this is for when you run the config.queue boot
queue_config = {'queue': 'run/posts', 'sleep': 10}
queue_handlers = ['app.handlers.index']
The @queue_config@ variable is read by the @config/queue.py@ file for the
@QueueReceiver@ and the @queue_handlers@ is fed to the @Router@ as described
above.
h2. Writing The Index Handler
You now have to write a new handler that is in @app/handlers/index.py@ so that
this @config.queue@ boot setup will load it and run it whenever a message hits
the @run/queue@. Here's the code:
from lamson import queue
from lamson.routing import route, stateless
import logging
@route("(post_name)@osb\\.(host)")
@stateless
def START(message, post_name=None, host=None):
logging.debug("Got message from %s", message['from'])
This simple demonstration will just log what messages it receives so you can
make sure it is working.
There are two points to notice about this handler. First, it is marked
@stateless@ because it will run independent of the regular Lamson server, and
you don't want its parallel operations to interfere with your normal server's
state operations. Second, it uses a @Router.defaults@ named @post_name@ that
you would add to your @config.settings.router_defaults@.
Once you have all this slightly complicated setup done you are ready to test
it.
bq. Also note that the examples in the "source releases":/releases/ have code
that does a deferred queue similar to this. Go look there for more code to
steal.
h2. Running Your Queue Receiver
Run your logger and lamson server like normal:
$ lamson log
$ lamson start
Next, go look in your logs and make sure it works by running your unit
tests:
$ nosetests
................
----------------------------------------------------------------------
Ran 16 tests in 1.346s
OK
Your logs should look normal, but now you should see some files in the
@run/posts/new@ directory:
$ ls run/posts/new/
1244080328.M408474P3147Q4.mycomputer.local
That's the results of your @POSTING@ handler putting the messages it receives
into your @run/posts@ maildir queue.
Finally, you'll want to run your queue receiver:
If you're running the code given above then you should see this in the
@logs/lamson.log@ file:
...
DEBUG:root:Sleeping for 10 seconds...
DEBUG:root:Pulled message with key:
'1244080328.M408474P3147Q4.zed-shaws-macbook.local' off
DEBUG:root:Message received from Peer: 'run/posts', From:
'sender-1244080328.22@sender.com', to To
['test.blog.1244080328@osb.test.com'].
DEBUG:root:Got message from sender-1244080328.22@sender.com
DEBUG:root:Message to test.blog.1244080328@osb.test.com was handled by
app.handlers.index.START
Which means your queue receiver is running. You could *in theory* run as many
of these as you wanted, as long as their handlers are stateless.
When you're done you can stop the whole setup with the following command:
$ lamson stop -ALL run
Stopping processes with the following PID files:
['run/log.pid', 'run/queue.pid', 'run/smtp.pid']
Attempting to stop lamson at pid 3092
Attempting to stop lamson at pid 3157
Attempting to stop lamson at pid 3096
h2. Further Advanced Usage
This configuration is debatable whether it is very usable or not, but it works
and will improve as the project continues. To give you some ideas of what you
can do with it:
# Defer activity to other machines or processes.
# Receive messages from other mail systems that know maildir.
# Deliver messages to other maildir aware systems.
# Process messages from a web application, and possibly even generic work.
It might also be possible to actually make your state functions transition to
the queue handler states by simply having the function return the
@module.FUNCTION@ that should be next. Take care with this though as it means
your end user's actions are effectively blocked for that event until the next
run of the queue receiver.
h2. Call For Suggestions
Feel free to offer suggestions in improving this setup (or even better code).
lamson-1.0pre11/doc/lamsonproject.org/output/docs/deploying_lamson.html 0000644 0000765 0000024 00000030331 11242710571 026000 0 ustar zedshaw staff
LamsonProject: Deploying Lamson Level 1
These instructions will teach you how to setup a completely clean Python 2.6
installation, a virtualenv with lamson, and all the gear needed to run the
oneshotblog.com software on your machine.
Most of these instructions could be easily turned into an automated script,
which may happen in the future. For now it is meant to teach you about the
typically dirty details involved in setting up a system for the first time. It
also tries to avoid various problems with different operating systems, so let
me know how it works for you.
A Warning
Deploying server software is a notoriously nasty process, especially the first
10 or 20 times. Most operating systems do their best to enforce completely
arbitrary restrictions on your file layouts and configurations, and every
system has different arbitrary restrictions.
When you go through these instructions, make sure you stay awake and be ready
to delve into why a particular step might not work on your system. There’s a
good chance you missed something or that there’s something just slightly
different about your system that makes the step not work.
For example, in the parts of this document where I setup
oneshotblog.com I ran into a problem with
SpamBayes dying because it couldn’t iterate
a bsddb. Problem is this works just fine in the exact same setup on a CentOS
machine and was only dying on a MacOSX machine that I later tested. For
whatever reason, the exact same setup can’t run SpamBayes on OSX, even though
it can run with the stock Python 2.5 in OSX.
To solve the problem I just had to show you how to disable SpamBayes in the
oneshotblog.com code so you could test it. That’s just how deployment goes.
You get on a machine and start setting things up and then 2/3 of the way
through the configuration you find out that something doesn’t work.
Only choices are to work around the problem (like I did) or try to figure out
why your machine is different and fix it.
Step 0: Setup A Workplace
You’ll want a directory to do this in so that you don’t screw up your machine.
Here’s what I did:
$ mkdir deploy
$ cd deploy
$ export DEPLOY=$PWD
That last bit is so you can refer to this deployment directory with $DEPLOY
(which I’ll be using in the instructions from now on).
Step 1: Get Python
Many operating systems have old versions of Python, and even though Lamson
works with 2.6 or 2.5, you’ll probably want to get 2.6 for your deployment. If
your OS has 2.6 available then go ahead and install it.
If it doesn’t have the right Python version, then here’s how you can install it
from source and use it as your default Python. To do this, just punch in these
commands:
$ mkdir source
$ cd source
$ wget http://www.python.org/ftp/python/2.6.2/Python-2.6.2.tgz
$ tar -xvf Python-2.6.2.tgz
$ cd Python-2.6.2
$ ./configure --prefix=$DEPLOY
$ make
$ make install
After this you will have a bunch of new directories in $DEPLOY:
$ ls $DEPLOY
bin include lib share source
Finally, you just have to put this new bin directory into your $PATH:
$ export PATH=$DEPLOY/bin:$PATH
... then you just try it out to make sure that you have the right one:
Python-2.6.2 $ which python
$DEPLOY/deploy/bin/python
Python-2.6.2 $ python
Python 2.6.2 (r262:71600, Jun 8 2009, 00:44:56)
[GCC 4.0.1 (Apple Inc. build 5490)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> ^D
That’s it, you’ll now be able to use this Python when you need to run your
Lamson server, and setup a virtualenv (coming next) so that you’re walled off
from the rest of the system.
Operating system fanatics will scoff at putting the python install in this
directory, so if you want you can just install it to the default /usr/local on
your system and deal with all the various clashes and conflicts you’ll have,
especially if you are on an MacOSX machine.
Step 2: Install VirtualEnv
Now we need to create a “virtual environment” to install all your software. To
do this we’ll need easy_install installed to your $DEPLOY directory:
$ cd $DEPLOY/source
$ wget http://peak.telecommunity.com/dist/ez_setup.py
$ python ez_setup.py
$ which easy_install
$DEPLOY/bin/easy_install
As you can see, you now have a clean install of easy_install in your fresh
$DEPLOY/bin directory for you to use. Now you need to install virtualenv:
$ easy_install --prefix $DEPLOY virtualenv
$ which virtualenv
$DEPLOY/bin/virtualenv
Make sure you use --prefix $DEPLOY above or you’ll install things into
the default system setup even though easy_install is clearly and obviously
running from a Python in a totally different location so easy_install should
know that.
Step 3: Create Your VirtualEnv
With that you are ready to setup your virtual environment which will house your
Lamson setup and fill it with the gear you need.
First up is getting your virtualenv created and activated:
$ cd $DEPLOY
$ virtualenv LAMSON
New python executable in LAMSON/bin/python
Installing setuptools............done.
$ cd LAMSON
$ . bin/activate
That’s pretty simple, and it tells you clearly that you are using the LAMSON
virtualenv. It prepends that to your currently prompt, so your prompt may look
different.
After that we can use easy_install to install our packages to this LAMSON
virtual env. Keep in mind that these packages will be in $DEPLOY/LAMSON, so
they won’t infect your regular $DEPLOY setup.
$ cd $DEPLOY/LAMSON
$ easy_install lamson
After that, you have lamson installed and ready to go, and you can install
anything you want, but there is one catch:
You MUST be in the $DEPLOY/LAMSON directory or easy_install barfs
complaining that the package is not there.
Step 4: Making Sure It Works
All of this setup is pointless if you can’t get back to it later, so exit your
terminal completely and start a new one so you can do this:
$ cd projects/lamson/deploy/
$ export DEPLOY=$PWD
$ export PATH=$DEPLOY/bin:$PATH
$ cd $DEPLOY/LAMSON
$ . bin/activate
(LAMSON) $ which python
$DEPLOY/deploy/LAMSON/bin/python
(LAMSON) $ which easy_install
$DEPLOY/deploy/LAMSON/bin/easy_install
(LAMSON) $ which lamson
$DEPLOY/deploy/LAMSON/bin/lamson
(LAMSON) $ cd $DEPLOY
(LAMSON) $ lamson help
If you can do all that, then you know you’ve got the setup going, now you just
need a little shell script to kick this all into gear automatically:
#!/bin/sh
export DEPLOY=$PWD
export PATH=$DEPLOY/bin:$PATH
cd $1
source bin/activate
cd $DEPLOY
To use this script, you just do this:
$ cd projects/lamson/deploy
$ . activate LAMSON
With that you have a fully ready to go setup that’s not using your normal
system’s Python at all, has Python 2.6 installed, a fully virtualenv, and the
start of your lamson setup.
This document is very fresh, so send me feedback on your experience with
running through it. Make sure you tell me what system you are on and that you
ran each command exactly when you do.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/deploying_lamson.txt 0000644 0000765 0000024 00000016746 11242710567 025676 0 ustar zedshaw staff Title: Deploying Lamson Level 1
These instructions will teach you how to setup a completely clean Python 2.6
installation, a virtualenv with lamson, and all the gear needed to run the
"oneshotblog.com":http://oneshotblog.com/ software on your machine.
You can then "read how to install oneshotblog":/docs/deploying_oneshotblog.html
for yourself.
Most of these instructions could be easily turned into an automated script,
which may happen in the future. For now it is meant to teach you about the
typically dirty details involved in setting up a system for the first time. It
also tries to avoid various problems with different operating systems, so let
me know how it works for you.
h2. A Warning
Deploying server software is a notoriously nasty process, especially the first
10 or 20 times. Most operating systems do their best to enforce completely
arbitrary restrictions on your file layouts and configurations, and every
system has different arbitrary restrictions.
When you go through these instructions, make sure you stay awake and be ready
to delve into why a particular step might not work on your system. There's a
good chance you missed something or that there's something just slightly
different about your system that makes the step not work.
For example, in the parts of this document where I setup
"oneshotblog.com":http://oneshotblog.com/ I ran into a problem with
"SpamBayes":http://spambayes.sourceforge.net/ dying because it couldn't iterate
a bsddb. Problem is this works just fine in the exact same setup on a CentOS
machine and was only dying on a MacOSX machine that I later tested. For
whatever reason, the exact same setup can't run SpamBayes on OSX, even though
it can run with the stock Python 2.5 in OSX.
To solve the problem I just had to show you how to disable SpamBayes in the
oneshotblog.com code so you could test it. That's just how deployment goes.
You get on a machine and start setting things up and then 2/3 of the way
through the configuration you find out that something doesn't work.
Only choices are to work around the problem (like I did) or try to figure out
why your machine is different and fix it.
h2. Step 0: Setup A Workplace
You'll want a directory to do this in so that you don't screw up your machine.
Here's what I did:
$ mkdir deploy
$ cd deploy
$ export DEPLOY=$PWD
That last bit is so you can refer to this deployment directory with $DEPLOY
(which I'll be using in the instructions from now on).
h2. Step 1: Get Python
Many operating systems have old versions of Python, and even though Lamson
works with 2.6 or 2.5, you'll probably want to get 2.6 for your deployment. If
your OS has 2.6 available then go ahead and install it.
If it doesn't have the right Python version, then here's how you can install it
from source and use it as your default Python. To do this, just punch in these
commands:
$ mkdir source
$ cd source
$ wget http://www.python.org/ftp/python/2.6.2/Python-2.6.2.tgz
$ tar -xvf Python-2.6.2.tgz
$ cd Python-2.6.2
$ ./configure --prefix=$DEPLOY
$ make
$ make install
After this you will have a bunch of new directories in $DEPLOY:
$ ls $DEPLOY
bin include lib share source
Finally, you just have to put this new bin directory into your $PATH:
$ export PATH=$DEPLOY/bin:$PATH
... then you just try it out to make sure that you have the right one:
Python-2.6.2 $ which python
$DEPLOY/deploy/bin/python
Python-2.6.2 $ python
Python 2.6.2 (r262:71600, Jun 8 2009, 00:44:56)
[GCC 4.0.1 (Apple Inc. build 5490)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> ^D
That's it, you'll now be able to use this Python when you need to run your
Lamson server, and setup a virtualenv (coming next) so that you're walled off
from the rest of the system.
bq. Operating system fanatics will scoff at putting the python install in this
directory, so if you want you can just install it to the default /usr/local on
your system and deal with all the various clashes and conflicts you'll have,
especially if you are on an MacOSX machine.
h2. Step 2: Install VirtualEnv
Now we need to create a "virtual environment" to install all your software. To
do this we'll need easy_install installed to your $DEPLOY directory:
$ cd $DEPLOY/source
$ wget http://peak.telecommunity.com/dist/ez_setup.py
$ python ez_setup.py
$ which easy_install
$DEPLOY/bin/easy_install
As you can see, you now have a clean install of easy_install in your fresh
$DEPLOY/bin directory for you to use. Now you need to install @virtualenv@:
$ easy_install --prefix $DEPLOY virtualenv
$ which virtualenv
$DEPLOY/bin/virtualenv
bq. Make sure you use @--prefix $DEPLOY@ above or you'll install things into
the default system setup even though easy_install is clearly and obviously
running from a Python in a totally different location so easy_install should
know that.
h2. Step 3: Create Your VirtualEnv
With that you are ready to setup your virtual environment which will house your
Lamson setup and fill it with the gear you need.
First up is getting your virtualenv created and activated:
$ cd $DEPLOY
$ virtualenv LAMSON
New python executable in LAMSON/bin/python
Installing setuptools............done.
$ cd LAMSON
$ . bin/activate
That's pretty simple, and it tells you clearly that you are using the LAMSON
virtualenv. It prepends that to your currently prompt, so your prompt may look
different.
After that we can use easy_install to install our packages to this LAMSON
virtual env. Keep in mind that these packages will be in $DEPLOY/LAMSON, so
they won't infect your regular $DEPLOY setup.
$ cd $DEPLOY/LAMSON
$ easy_install lamson
After that, you have lamson installed and ready to go, and you can install
anything you want, but there is one catch:
bq. You *MUST* be in the $DEPLOY/LAMSON directory or easy_install barfs
complaining that the package is not there.
h2. Step 4: Making Sure It Works
All of this setup is pointless if you can't get back to it later, so exit your
terminal completely and start a new one so you can do this:
$ cd projects/lamson/deploy/
$ export DEPLOY=$PWD
$ export PATH=$DEPLOY/bin:$PATH
$ cd $DEPLOY/LAMSON
$ . bin/activate
(LAMSON) $ which python
$DEPLOY/deploy/LAMSON/bin/python
(LAMSON) $ which easy_install
$DEPLOY/deploy/LAMSON/bin/easy_install
(LAMSON) $ which lamson
$DEPLOY/deploy/LAMSON/bin/lamson
(LAMSON) $ cd $DEPLOY
(LAMSON) $ lamson help
If you can do all that, then you know you've got the setup going, now you just
need a little shell script to kick this all into gear automatically:
#!/bin/sh
export DEPLOY=$PWD
export PATH=$DEPLOY/bin:$PATH
cd $1
source bin/activate
cd $DEPLOY
To use this script, you just do this:
$ cd projects/lamson/deploy
$ . activate LAMSON
With that you have a fully ready to go setup that's not using your normal
system's Python at all, has Python 2.6 installed, a fully virtualenv, and the
start of your lamson setup.
h2. Conclusion
Your next step is to try and setup "oneshotblog":http://oneshotblog.com/ using
the "instructions I've written":/docs/deploying_oneshotblog.html to follow
these instructions.
This document is very fresh, so send me feedback on your experience with
running through it. Make sure you tell me what system you are on and that you
ran each command exactly when you do.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/deploying_lamson_level_2.html 0000644 0000765 0000024 00000007675 11243030345 027422 0 ustar zedshaw staff
LamsonProject: Deploying Lamson Level 2
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
These instructions follow from Deploying Lamson Level 1 and you should follow those first before attempting these. If you run into
problems with these instructions, then email the lamson@librelist.com mailing list for help.
Step 5: Setting Up The OneShotBlog
Let’s see if we can setup the OneShotBlog example from the Lamson source the
way it is on the oneshotblog.com site. We’ll need a
few more modules installed with easy_install:
Let’s grab the 0.9.3 source from PyPI so we can get at the OSB example source:
$ cd $DEPLOY/source
$ wget http://pypi.python.org/packages/source/l/lamson/lamson-0.9.3.tar.gz
$ tar -xzf lamson-0.9.3.tar.gz
$ cd lamson-0.9.3/examples/osb
Now we hit a slight snag. OSB is using
SpamBayes to do spam filtering, but you
probably will have a broken setup and would need to configure a ton of stuff to
get it working. For now we’re, just going to cheat, since it looks like
SpamBayes has problems with trying to iterate the keys in a bsddb under some
Python builds. To avoid the problem, we’re just going to edit
app/handlers/comment.py to remove the line with spam_filter:
@route("(user_id)-AT-(domain)-(post_name)-comment@(host)")
# DELETE THIS LINE IN app/handlers/comments.py
@spam_filter(SPAM['db'], SPAM['rc'], SPAM['queue'], next_state=SPAMMING)
def START(message, user_id=None, post_name=None, host=None, domain=None):
comment.attach_headers(message, user_id, post_name, domain)
confirmation.send(relay, "comment", message, "mail/comment_confirm.msg", locals())
return CONFIRMING
The spam filtering does work, but SpamBayes is difficult to get working in such
a small test run.
You are now sitting in the OSB example code, so you can fire up the logger
server and run the unit tests to make sure everything is working:
..............
======================================================================
FAIL: handlers.comments_tests.test_spam_sent_by_unconfirmed_user
----------------------------------------------------------------------
Traceback (most recent call last):
...
-------------------- >> begin captured logging << --------------------
root: WARNING: Attempt to post to user 'spamtester@somehost.com' but user doesn't exist.
--------------------- >> end captured logging << ---------------------
======================================================================
FAIL: handlers.comments_tests.test_spam_sent_by_confirmed_user
----------------------------------------------------------------------
Traceback (most recent call last):
...
-------------------- >> begin captured stdout << ---------------------
run/posts count after dever 1
run/posts count after dever 2
--------------------- >> end captured stdout << ----------------------
-------------------- >> begin captured logging << --------------------
root: WARNING: Attempt to post to user 'spamtester@somehost.com' but user doesn't exist.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 25 tests in 1.363s
Those are just fine since you don’t have PyEnchant installed and aren’t using
the spam filtering.
Step 6: Run OneShotBlog Example
Now you’re running the logger server and have your unit tests going, and
hopefully you can fix anything that you run into by now. All you need now is
to run the whole setup and try it out:
With all this gear running you should be able to look in the logs/lamson.log
and logs/logger.log to see what’s going on. You’ll see the following
activity:
Forwarding receiver taking mail that couldn’t be delivered and forwarding it to the logger server.
The queue receiver pulling messages off run/posts and either delivering them as comments or updating the index.
The rest of lamson processing mail and doing its job of feeding these two or just sending emails.
If you want to see the configuration for these two other servers look in config/queue.py and
config/forward.py or better yet, diff them against config/boot.py to see what’s really different.
You should also check to see that they are really running:
Now we get to play with it. Lamson comes with a web server that you can run to
do simple testing so start up a second window/terminal and do this:
$ cd projects/lamson/deploy/
$ . activate LAMSON
(LAMSON) $ cd source/lamson-0.9.3/examples/osb/
(LAMSON) $ lamson web -basedir app/data
Starting server on 127.0.0.1:8888 out of directory 'app/data'
Now hit http://localhost:8888/ with your browser and
see the junk left over from your test runs. Most of those posts won’t actually
exist, so let’s make a fake one for now.
You can forget about this web server window for now, and go back to your LAMSON
window to do this with mutt:
mutt -F muttrc to get it going with a fake setup.
Send an email to first.blog@oneshotblog.com (m is the key).
You’ll get a confirmation back, reply to it.
You’ll get a welcome message, but this message isn’t in the index yet. You can go look at it directly though.
Send another email, this time to my.new.post@oneshotblog.com.
No confirmation this time, just a message saying it was completed.
Now go look at the index (might take a few seconds, up to 10).
Click on the post title and go look at it.
Right click on the [send comment] link and copy the email address.
Go back to mutt and send an email to that address, this will post a comment to that post.
Reply to the comment confirmation email, this should be the only one you get.
In about 10 seconds you’ll see your comment show up.
With that you have fully tested out the OneShotBlog example. All that remains
would be a full deployment in a for-real situation, which is what we’ll do
next.
Step 8: Running On Port 25 For Real
The only problem with testing out the OneShotBlog with your own email client is
that you need to trick your computer into thinking your localhost address is
also oneshotblog.com. To do that, open your /etc/hosts file and make whatever
changes you need to have localhost be oneshotblog.com also.
You’ll know you’ve got it right when you can point your browser at
oneshotblog.com and see your lamson web window
display log messages showing you click around.
Remember to undo this or you might be annoyed later.
Next, you’ll need to stop lamson and restart it to use port 25. This will be a
problem if you have another server running on port 25, so make sure you turn
that server off for now.
I hope you aren’t doing this on a live site where that’s a problem.
$ lamson stop
$ vim config/settings.py
At this point you’ll want to change the receiver_config to look like this in
config/settings.py:
Then you’ll want to restart lamson so that it drops privilege to your user
after grabbing that port. Easiest way to find out what your user id (uid) and
group id (gid) are is to use Python:
Python 2.6.2 (r262:71600, Jun 8 2009, 00:44:56)
[GCC 4.0.1 (Apple Inc. build 5490)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import os
>>> os.getuid()
500
>>> os.getgid()
500
>>> ^D
This shows that I’m uid 500 and gid 500, so now I can start my server:
That last command just makes sure that lamson didn’t accidentally change the
permission of a stray file or queue to root.
With that you should be running your lamson server as your user rather than as
root, but still bound to port 25. Here’s how to check:
$ ps aux | grep lamson | grep root
If you don’t see anything printed out, then you’re safe. If you see something
running as root, then you’ve got some work to do.
Conclusion
I’ll leave it to you to actually get your mail client to talk to this
OneShotBlog and working. If you have problems, look at all the files in
logs/ and also use the lamson queue command to inspect the different queues
in the run/ directory.
At this point, you should know enough about setting up a lamson server and
configuring a real application (warts and all). You should also have learned
how to get a clean Python installation that you can use no matter what your
native OS does to Python, even if it’s retarded and renames python to Python
for no apparent reason.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/deploying_oneshotblog.txt 0000644 0000765 0000024 00000022614 11242710431 026705 0 ustar zedshaw staff Title: Deploying OneShotBlog
These instructions follow from "Deploying Lamson Level 1":/docs/deploying_lamson.html and you should follow those first before attempting these. If you run into
problems with these instructions, then "email the lamson@librelist.com":mailto:lamson@librelist.com mailing list for help.
h2. Step 5: Setting Up The OneShotBlog
Let's see if we can setup the OneShotBlog example from the Lamson source the
way it is on the "oneshotblog.com":http://oneshotblog.com site. We'll need a
few more modules installed with easy_install:
Let's grab the 0.9.3 source from "PyPI":http://pypi.python.org/pypi/lamson/0.9.3 so we can get at the OSB example source:
$ cd $DEPLOY/source
$ wget http://pypi.python.org/packages/source/l/lamson/lamson-0.9.3.tar.gz
$ tar -xzf lamson-0.9.3.tar.gz
$ cd lamson-0.9.3/examples/osb
Now we hit a slight snag. OSB is using
"SpamBayes":http://spambayes.sourceforge.net/ to do spam filtering, but you
probably will have a broken setup and would need to configure a ton of stuff to
get it working. For now we're, just going to cheat, since it looks like
SpamBayes has problems with trying to iterate the keys in a bsddb under *some*
Python builds. To avoid the problem, we're just going to edit
@app/handlers/comment.py@ to remove the line with @spam_filter@:
@route("(user_id)-AT-(domain)-(post_name)-comment@(host)")
# DELETE THIS LINE IN app/handlers/comments.py
@spam_filter(SPAM['db'], SPAM['rc'], SPAM['queue'], next_state=SPAMMING)
def START(message, user_id=None, post_name=None, host=None, domain=None):
comment.attach_headers(message, user_id, post_name, domain)
confirmation.send(relay, "comment", message, "mail/comment_confirm.msg", locals())
return CONFIRMING
The spam filtering does work, but SpamBayes is difficult to get working in such
a small test run.
You are now sitting in the OSB example code, so you can fire up the logger
server and run the unit tests to make sure everything is working:
..............
======================================================================
FAIL: handlers.comments_tests.test_spam_sent_by_unconfirmed_user
----------------------------------------------------------------------
Traceback (most recent call last):
...
-------------------- >> begin captured logging << --------------------
root: WARNING: Attempt to post to user 'spamtester@somehost.com' but user doesn't exist.
--------------------- >> end captured logging << ---------------------
======================================================================
FAIL: handlers.comments_tests.test_spam_sent_by_confirmed_user
----------------------------------------------------------------------
Traceback (most recent call last):
...
-------------------- >> begin captured stdout << ---------------------
run/posts count after dever 1
run/posts count after dever 2
--------------------- >> end captured stdout << ----------------------
-------------------- >> begin captured logging << --------------------
root: WARNING: Attempt to post to user 'spamtester@somehost.com' but user doesn't exist.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 25 tests in 1.363s
Those are just fine since you don't have PyEnchant installed and aren't using
the spam filtering.
h2. Step 6: Run OneShotBlog Example
Now you're running the logger server and have your unit tests going, and
hopefully you can fix anything that you run into by now. All you need now is
to run the whole setup and try it out:
With all this gear running you should be able to look in the @logs/lamson.log@
and @logs/logger.log@ to see what's going on. You'll see the following
activity:
* Forwarding receiver taking mail that couldn't be delivered and forwarding it to the logger server.
* The queue receiver pulling messages off run/posts and either delivering them as comments or updating the index.
* The rest of lamson processing mail and doing its job of feeding these two or just sending emails.
bq. If you want to see the configuration for these two other servers look in @config/queue.py@ and
@config/forward.py@ or better yet, diff them against @config/boot.py@ to see what's really different.
You should also check to see that they are really running:
h2. Step 7: Playing With OneShotBlog
Now we get to play with it. Lamson comes with a web server that you can run to
do simple testing so start up a second window/terminal and do this:
$ cd projects/lamson/deploy/
$ . activate LAMSON
(LAMSON) $ cd source/lamson-0.9.3/examples/osb/
(LAMSON) $ lamson web -basedir app/data
Starting server on 127.0.0.1:8888 out of directory 'app/data'
Now hit "http://localhost:8888/":http://localhost:8888/ with your browser and
see the junk left over from your test runs. Most of those posts won't actually
exist, so let's make a fake one for now.
You can forget about this web server window for now, and go back to your LAMSON
window to do this with mutt:
# mutt -F muttrc to get it going with a fake setup.
# Send an email to first.blog@oneshotblog.com (m is the key).
# You'll get a confirmation back, reply to it.
# You'll get a welcome message, but this message isn't in the index yet. You can go look at it directly though.
# Send *another* email, this time to my.new.post@oneshotblog.com.
# No confirmation this time, just a message saying it was completed.
# *Now* go look at the index (might take a few seconds, up to 10).
# Click on the post title and go look at it.
# Right click on the [send comment] link and copy the email address.
# Go back to mutt and send an email to that address, this will post a comment to that post.
# Reply to the comment confirmation email, this should be the only one you get.
# In about 10 seconds you'll see your comment show up.
With that you have fully tested out the OneShotBlog example. All that remains
would be a full deployment in a for-real situation, which is what we'll do
next.
h2. Step 8: Running On Port 25 For Real
The only problem with testing out the OneShotBlog with your own email client is
that you need to trick your computer into thinking your localhost address is
also oneshotblog.com. To do that, open your /etc/hosts file and make whatever
changes you need to have localhost be oneshotblog.com also.
You'll know you've got it right when you can point your browser at
"oneshotblog.com":http://oneshotblog.com/ and see your @lamson web@ window
display log messages showing you click around.
bq. Remember to undo this or you might be annoyed later.
Next, you'll need to stop lamson and restart it to use port 25. This will be a
problem if you have another server running on port 25, so make sure you turn
that server off for now.
bq. I hope you aren't doing this on a live site where that's a problem.
$ lamson stop
$ vim config/settings.py
At this point you'll want to change the receiver_config to look like this in
@config/settings.py@:
Then you'll want to restart lamson so that it drops privilege to your user
after grabbing that port. Easiest way to find out what your user id (uid) and
group id (gid) are is to use Python:
Python 2.6.2 (r262:71600, Jun 8 2009, 00:44:56)
[GCC 4.0.1 (Apple Inc. build 5490)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import os
>>> os.getuid()
500
>>> os.getgid()
500
>>> ^D
This shows that I'm uid 500 and gid 500, so now I can start my server:
That last command just makes sure that lamson didn't accidentally change the
permission of a stray file or queue to @root@.
With that you should be running your lamson server as your user rather than as
root, but still bound to port 25. Here's how to check:
$ ps aux | grep lamson | grep root
If you don't see anything printed out, then you're safe. If you see something
running as root, then you've got some work to do.
h2. Conclusion
I'll leave it to you to actually get your mail client to talk to this
OneShotBlog and working. If you have problems, look at all the files in
@logs/@ and also use the @lamson queue@ command to inspect the different queues
in the @run/@ directory.
At this point, you should know enough about setting up a lamson server and
configuring a real application (warts and all). You should also have learned
how to get a clean Python installation that you can use no matter what your
native OS does to Python, even if it's retarded and renames python to Python
for no apparent reason.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/faq.html 0000644 0000765 0000024 00000047514 11242712467 023225 0 ustar zedshaw staff
LamsonProject: Frequently Asked Questions
Lamson is a pure Python SMTP server designed to create robust and complex mail
applications in the style of modern web frameworks such as Django. Unlike
traditional SMTP servers like Postfix or Sendmail, Lamson has all the features
of a web application stack (ORM, templates, routing, handlers, state machines,
Python) without needing to configure alias files, run newaliases, or juggle
tons of tiny fragile processes. Lamson also plays well with other web
frameworks and Python libraries.
Where does the name “Lamson” come from?
“Lamson Tubes” is a colloquial name for Pneumatic Tubes which were used last
century to deliver mail, packages, and hazardous material to the corporate
world. They are still in use today.
What kind of applications do you envision being built on top of Lamson?
Why, spam of course (like I could stop that). As well as spam fighters,
greylisters, “campaign management” applications, mail firewalls, mime strippers
(for those Exchange file shares), help desk support applications, games,
mailing lists ('cause everyone loves writing those), and even SMTP portions of
just about any web site.
A few examples of applications that are actually written using Lamson (with
source included in the source releases
How do I install lamson on a Debian or CentOS server?
Debian and CentOS are notorious for being dinosaurs. Both distributions of
Linux suffer from the false rationale that older software is more stable and
secure. The reality is that the stability or security of a piece of software
is not a function of its age, and in many cases the newer versions of software
will typically fix many stability and performance problems. Despite this fact,
these two variants of Linux are notorious for back-porting patches from later
versions to older versions rather than just using the newer version.
In order to help people who need to run a modern piece of software on their
antiquated operating systems, I’ve written an extensive document on how to
Deploy Lamson that tells you how to do it in a way
that should work reliably on most Unix platforms, even if they have an older
version of Python.
Is there a mailing list?
Yes, and it’s written in Lamson. You can send an email to lamson@librelist.com
and you’ll be able to join. The code for this mailing list system is also included in the source releases
so you can learn from it.
Is there an IRC channel?
Yes, and it has relatively low traffic. You can join #librelist on irc.freenode.org.
How do I report a BUG/Feature/Question in Lamson?
Currently the best place to report a Lamson bug is to the lamson@librelist.com
mailing list. If you don’t want to subscribe to the list to just report a bug, then you can
contact me directly to report it.
How do I work on Lamson?
I currently do all the work on Lamson myself. I don’t want to discourage contributions, but I’ve
found that when a project is small and just getting started it’s best to keep it under the control
of one person.
I try to make “1.0” what most other people would consider 80% complete. This
lets people get it and work with it, and then I can refine it for the 2.0
release with 100% of features people actually use.
With this in mind, I’m planning on the 1.0 release to be based on the sample
applications and my own applications and include the smallest set of features.
The most important part of the 1.0 release will be good documentation and bug
free high quality code.
What are Lamson’s current features?
Lamson is currently running a handful of sites and is almost ready for a 1.0 release.
It has these high level features:
Very good bounce detection and analysis.
Spam blocking.
Get an application going quickly with the “lamson gen” command.
Handle mail for arbitrary hosts and addresses using simple regex routing.
Process requests including full access to Python’s complete MIME email libraries.
Absolutely awesome full conversion to Unicode including complete cleaning of incoming mail.
Routing requests in a standard web application framework style based on addresses.
Processing mail messages (requests) in either simple generic handlers, or in more complex and robust state machines.
Use any Python database storage you want.
Use of Jinja2 or Mako templates, with Jinja2 as the default.
Craft and send plain text or HTML email including attachments, with a great HTML mail generation API that even “knits” in CleverCSS.
Unit test helpers for having a conversation, checking spelling, etc.
Defer processing to Maildir queues for offline handling of bigger tasks.
Extensive logging and development tools for debugging your email applications.
Mutt configurations to fake out mutt so that it talks to your server.
100% code coverage in the unit tests.
Isn’t Postfix/Sendmail/Exim Faster?
That is a tough question to answer actually. If all you need to do is receive
and deliver email then a well established traditional email server like Postfix
or Exim (please don’t use Sendmail) is the way to go. Hands down these servers
are the fastest and the best at this job.
However, if you need to actually do something smart with your email, like
manage many mailing lists or handle support requests, then these servers are
definitely slower.
The reason is they require that you configure them to take messages they’ve
already received and hand them to a separate process like a Perl, Python, or
Ruby script. This separate process then has to parse the message again, do
its job without stepping on any other processes that might be running (that
means locks), and then send response messages back to the server for even more
SMTP parsing.
With the triple and sometimes quadruple MIME parsing, the heavy weight
processes, the difficult to manage locking, and the additional configuration
headaches, there’s no way traditional mail servers beat Lamson in speed.
Lamson only processes a message once, maybe twice if you defer to a queue.
Once the message is parsed you get full access to Python immediately, without
spawning a separate process. Even if you defer to a queue, the Lamson dequeue
server stays resident and processes the queue without forking. You can even
run many dequeue servers on mulitple machines processing a shared Maildir if
you need the extra processing.
In the end, threads and function calls beats processes and pipes.
Why not use Sendmail’s Milter?
Sendmail has a protocol named “Milter” that lets you write a mail processing
server that acts as a sort of “slave” to the sendmail process. This protocol
is supported by at least Postfix as well, maybe other servers.
Feel free to go try Milter. When you’re done trying to figure out the protocol
from the dense C code, configure the m4 macros, find a decent milter protocol
library that doesn’t involve installing sendmail, and debugging the final
setup, then you can come back and have it easy with Lamson.
Why does Lamson send messages to a relay host?
Lamson doesn’t have to deliver to a relay host, but it is a smarter more
practical use of the technology.
Lamson is written in Python and does actually run slower than the established
mail servers. In addition, Lamson is hopefully doing something more than just
routing email around to people. It is probably processing messages, crafting
replies, querying databases, hitting REST interfaces, and all the other things
you’d want to do with a modern application. This takes time and resources and
are probably more valuable operations than just simple delivery.
For this reason, you want to use a dumb workhorse like Postfix to do your
actual delivery, and reserve the smart processing that has value for Lamson.
What about security?! Shouldn’t Lamson be 20 processes?
Have you ever asked why other mail servers are a bundle of a billion processes?
Why have one server receiving mail, another routing it, and another handing it
to users?
The answer is back in the 1970’s most mail was delivered to Unix users in their
home directories or similar files that required special access rights to
modify. Also in the 1970’s special ports like 25 for SMTP required root
access, which in the tiny Internet of the time meant that the server could be
“trusted”. These two realities of the time meant that to receive and deliver
mail at least some part of the system had to run as root. To keep things safe,
modern mail systems reduce the amount of time spent as the root user by
separating their functionality into different processes.
However, if you never have to deliver to a user, and all you ever do is process
mail and talk to other servers like RDBMS, then why do you need all this
privilege separation? Sites run just fine with systems running as one or two
processes without the complexity of some illogical privilege separation getting
in their way.
To put this into perspective, imagine that you were writing a Django
application and you were required to have a separate process for the HTTP
layers, the view layer, the model layer, the HTTP responses, and the RDBMS
access layers? Each one required a different user, a different configuration
file, and you needed another process just to keep them all sane. All of this
just so that if someone hacks into your HTTP server as root they supposedly
can’t cause any damage.
Yet, they are on your server as root after all.
In practice, you can run Lamson as a separate root process, and then use
another “dequeue server” to do the real processing, if you feel you need that
security.
But, consider delaying that decision until you absolutely need it, because the
security benefits aren’t worth the development and deployment hassles.
How come nobody thought of this before?
I don’t know why, since it did seem kind of simple. There’s at least one other
project written in Perl called qpsmtpd that does
something similar, and there may be more. If you know others feel free to
contact me and let me know.
Isn’t [Insert Random Java Mail Server] actually the first mail “framework”?
I get this quite frequently when I make the claim that Lamson is the first
email framework, and it may be true that there was a framework out there before
Lamson. The internet is a big place, so anything is possible. However, I
looked really hard and I couldn’t find a single modern mail framework. All
that existed were servers I could use to build a framework.
You see, around 2003 or 2004 the concept of “framework” changed. Before then
all you needed was a server with an extension API named so that it rhymed with
“Servlet”. As long as your server provided a way to drop a class into the
processing queue and let a programmer handle the request you could call that a
framework.
The usual end result for these servers is that you could use them to build a
framework if you wanted, but what you’d get is affectionately called a
“frankenstack”. You’d grab an ORM from here, a template system from there,
maybe a workflow engine, write a Maven or Ant script to manage it, and wire it
all together with some lame secret sauce code you think gives you a competitive
edge.
Then along came the modern frameworks like Django and Rails that included
everything you needed in a bundle that you could use right away. They had ORM,
templating, routing, higher level request processing, email support, REST
support, and anything else you might need for the 80% of your application you
don’t care about.
Some people prefer less of these defaults, some people more, but nearly
everyone who has to get a project done prefers more than just an extension API
so they can build their own framework.
Today if you try to claim Apache James is a
framework you’d be wrong. I could build a framework with it, but I could
just as easily build that same framework with Python, Ruby, sendmail and even
postfix. James and friends are just servers, not frameworks. In fact, my
experience with James and similar Java mail servers is they are much harder to
use than aliases+pipes in Postfix.
I now advocate that if your framework doesn’t at least support data, views, and
high level logic as first class entities then it’s just a server. You don’t
have to use ORM, any particular templating, or Finite State Machines like
Lamson does. You don’t even have to settle on only one way to do data, views,
and logic.
You must at least support data, views, and logic out of the box so your user
doesn’t have to go shopping at “APIs-R-Us” just to use your gear.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/faq.txt 0000644 0000765 0000024 00000033652 11242712465 023074 0 ustar zedshaw staff Title: Frequently Asked Questions
If a question is missing you can "email the lamson@librelist.com mailing
list":mailto:lamson@librelist.com about it and I'll answer.
h2. What is Lamson?
Lamson is a pure Python SMTP server designed to create robust and complex mail
applications in the style of modern web frameworks such as Django. Unlike
traditional SMTP servers like Postfix or Sendmail, Lamson has all the features
of a web application stack (ORM, templates, routing, handlers, state machines,
Python) without needing to configure alias files, run newaliases, or juggle
tons of tiny fragile processes. Lamson also plays well with other web
frameworks and Python libraries.
h2. Where does the name "Lamson" come from?
"Lamson Tubes" is a colloquial name for Pneumatic Tubes which were used last
century to deliver mail, packages, and hazardous material to the corporate
world. They are still in use today.
h2. What kind of applications do you envision being built on top of Lamson?
Why, spam of course (like I could stop that). As well as spam fighters,
greylisters, "campaign management" applications, mail firewalls, mime strippers
(for those Exchange file shares), help desk support applications, games,
mailing lists ('cause everyone loves writing those), and even SMTP portions of
just about any web site.
A few examples of applications that are actually written using Lamson (with
source included in the "source releases":/releases/
* "OneShotBlog":http://oneshotblog.com/
* "Librelist":http://librelist.com/
* "MyInboxIsNotA.TV":http://myinboxisnota.tv/
With many more to come.
h2. How do I install lamson on a Debian or CentOS server?
Debian and CentOS are notorious for being dinosaurs. Both distributions of
Linux suffer from the false rationale that older software is more stable and
secure. The reality is that the stability or security of a piece of software
is not a function of its age, and in many cases the newer versions of software
will typically fix many stability and performance problems. Despite this fact,
these two variants of Linux are notorious for back-porting patches from later
versions to older versions rather than just using the newer version.
In order to help people who need to run a modern piece of software on their
antiquated operating systems, I've written an extensive document on "how to
Deploy Lamson":/docs/deploying_lamson.html that tells you how to do it in a way
that should work reliably on most Unix platforms, even if they have an older
version of Python.
h2. Is there a mailing list?
Yes, and it's written in Lamson. You can send an email to "lamson@librelist.com":mailto:lamson@librelist.com
and you'll be able to join. The code for this mailing list system is also included in the "source releases":/releases/
so you can learn from it.
h2. Is there an IRC channel?
Yes, and it has relatively low traffic. You can join #librelist on irc.freenode.org.
h2. How do I report a BUG/Feature/Question in Lamson?
Currently the best place to report a Lamson bug is to the "lamson@librelist.com":mailto:lamson@librelist.com
mailing list. If you don't want to subscribe to the list to just report a bug, then you can
"contact me":/contact.html directly to report it.
h2. How do I work on Lamson?
I currently do all the work on Lamson myself. I don't want to discourage contributions, but I've
found that when a project is small and just getting started it's best to keep it under the control
of one person.
If you find bugs, then please report them to the "lamson@librelist.com":mailto:lamson@librelist.com mailing
list to "to me directly":/contact.html and I'll fix them up.
h2. How do I try out (install) Lamson?
Best way to do it is to have
"easy_install":http://peak.telecommunity.com/DevCenter/EasyInstall and simply
do:
$ sudo easy_install lamson
Then you'll get it installed and can play with it. Refer to the "getting
started":/docs/getting_started.html documentation for more information.
h2. How can I get the code to Lamson?
Lamson lives on Launchpad at "https://launchpad.net/lamson":https://launchpad.net/lamson where you can get the code via:
bzr branch lp:lamson
Bazaar may ask you to login, but it should still give you the source.
Refer to the "download instructions":/download.html for more information.
h2. What features are planned for the 1.0 release?
I try to make "1.0" what most other people would consider 80% complete. This
lets people get it and work with it, and then I can refine it for the 2.0
release with 100% of features people actually use.
With this in mind, I'm planning on the 1.0 release to be based on the sample
applications and my own applications and include the smallest set of features.
The most important part of the 1.0 release will be good documentation and bug
free high quality code.
h2. What are Lamson's current features?
Lamson is currently running a handful of sites and is almost ready for a 1.0 release.
It has these high level features:
* Very good bounce detection and analysis.
* Spam blocking.
* Get an application going quickly with the "lamson gen" command.
* Full set of "self-documented commands":/docs/lamson_commands.html for managing and developing a lamson application.
* Lots of "documentation":/docs/ with more to come.
* Sample applications for you to base your work on.
* Handle mail for arbitrary hosts and addresses using simple regex routing.
* Process requests including full access to Python's complete MIME email libraries.
* Absolutely awesome full conversion to Unicode including complete cleaning of incoming mail.
* Routing requests in a standard web application framework style based on addresses.
* Processing mail messages (requests) in either simple generic handlers, or in more complex and robust state machines.
* Use any Python database storage you want.
* Use of Jinja2 or Mako templates, with Jinja2 as the default.
* Craft and send plain text or HTML email including attachments, with a great HTML mail generation API that even "knits" in CleverCSS.
* Unit test helpers for having a conversation, checking spelling, etc.
* Defer processing to Maildir queues for offline handling of bigger tasks.
* Extensive logging and development tools for debugging your email applications.
* Mutt configurations to fake out mutt so that it talks to your server.
* 100% code coverage in the unit tests.
h2. Isn't Postfix/Sendmail/Exim Faster?
That is a tough question to answer actually. If all you need to do is receive
and deliver email then a well established traditional email server like Postfix
or Exim (please don't use Sendmail) is the way to go. Hands down these servers
are the fastest and the best at this job.
However, if you need to actually do something smart with your email, like
manage many mailing lists or handle support requests, then these servers are
definitely slower.
The reason is they require that you configure them to take messages they've
already received and hand them to a separate process like a Perl, Python, or
Ruby script. This separate process then has to parse the message *again*, do
its job without stepping on any other processes that might be running (that
means locks), and then send response messages back to the server for even more
SMTP parsing.
With the triple and sometimes quadruple MIME parsing, the heavy weight
processes, the difficult to manage locking, and the additional configuration
headaches, there's no way traditional mail servers beat Lamson in speed.
Lamson only processes a message once, maybe twice if you defer to a queue.
Once the message is parsed you get full access to Python immediately, without
spawning a separate process. Even if you defer to a queue, the Lamson dequeue
server stays resident and processes the queue without forking. You can even
run many dequeue servers on mulitple machines processing a shared Maildir if
you need the extra processing.
In the end, threads and function calls beats processes and pipes.
h2. Why not use Sendmail's Milter?
Sendmail has a protocol named "Milter" that lets you write a mail processing
server that acts as a sort of "slave" to the sendmail process. This protocol
is supported by at least Postfix as well, maybe other servers.
Feel free to go try Milter. When you're done trying to figure out the protocol
from the dense C code, configure the m4 macros, find a decent milter protocol
library that doesn't involve installing sendmail, and debugging the final
setup, then you can come back and have it easy with Lamson.
h2. Why does Lamson send messages to a relay host?
Lamson doesn't have to deliver to a relay host, but it is a smarter more
practical use of the technology.
Lamson is written in Python and does actually run slower than the established
mail servers. In addition, Lamson is hopefully doing something more than just
routing email around to people. It is probably processing messages, crafting
replies, querying databases, hitting REST interfaces, and all the other things
you'd want to do with a modern application. This takes time and resources and
are probably more valuable operations than just simple delivery.
For this reason, you want to use a dumb workhorse like Postfix to do your
actual delivery, and reserve the smart processing that has value for Lamson.
h2. What about security?! Shouldn't Lamson be 20 processes?
Have you ever asked why other mail servers are a bundle of a billion processes?
Why have one server receiving mail, another routing it, and another handing it
to users?
The answer is back in the 1970's most mail was delivered to Unix users in their
home directories or similar files that required special access rights to
modify. Also in the 1970's special ports like 25 for SMTP required root
access, which in the tiny Internet of the time meant that the server could be
"trusted". These two realities of the time meant that to receive and deliver
mail at least some part of the system had to run as root. To keep things safe,
modern mail systems reduce the amount of time spent as the root user by
separating their functionality into different processes.
However, if you never have to deliver to a user, and all you ever do is process
mail and talk to other servers like RDBMS, then why do you need all this
privilege separation? Sites run just fine with systems running as one or two
processes without the complexity of some illogical privilege separation getting
in their way.
To put this into perspective, imagine that you were writing a Django
application and you were required to have a separate process for the HTTP
layers, the view layer, the model layer, the HTTP responses, and the RDBMS
access layers? Each one required a different user, a different configuration
file, and you needed another process just to keep them all sane. All of this
just so that if someone hacks into your HTTP server as root they supposedly
can't cause any damage.
Yet, they are on your server as root after all.
In practice, you can run Lamson as a separate root process, and then use
another "dequeue server" to do the real processing, if you feel you need that
security.
But, consider delaying that decision until you absolutely need it, because the
security benefits aren't worth the development and deployment hassles.
h2. How come nobody thought of this before?
I don't know why, since it did seem kind of simple. There's at least one other
project written in Perl called "qpsmtpd":http://smtpd.develooper.com/ that does
something similar, and there may be more. If you know others feel free to
"contact me":/contact.html and let me know.
h2. Isn't [Insert Random Java Mail Server] actually the first mail "framework"?
I get this quite frequently when I make the claim that Lamson is the first
email framework, and it may be true that there was a framework out there before
Lamson. The internet is a big place, so anything is possible. However, I
looked really hard and I couldn't find a single *modern* mail framework. All
that existed were servers I could use to build a framework.
You see, around 2003 or 2004 the concept of "framework" changed. Before then
all you needed was a server with an extension API named so that it rhymed with
"Servlet". As long as your server provided a way to drop a class into the
processing queue and let a programmer handle the request you could call that a
framework.
The usual end result for these *servers* is that you could use them to build a
framework if you wanted, but what you'd get is affectionately called a
"frankenstack". You'd grab an ORM from here, a template system from there,
maybe a workflow engine, write a Maven or Ant script to manage it, and wire it
all together with some lame secret sauce code you think gives you a competitive
edge.
Then along came the modern frameworks like Django and Rails that included
everything you needed in a bundle that you could use right away. They had ORM,
templating, routing, higher level request processing, email support, REST
support, and anything else you might need for the 80% of your application you
don't care about.
Some people prefer less of these defaults, some people more, but nearly
everyone who has to get a project done prefers more than just an extension API
so they can build their own framework.
Today if you try to claim "Apache James":http://james.apache.org/ is a
framework you'd be wrong. I could *build* a framework with it, but I could
just as easily build that same framework with Python, Ruby, sendmail and even
postfix. James and friends are just servers, not frameworks. In fact, my
experience with James and similar Java mail servers is they are much harder to
use than aliases+pipes in Postfix.
I now advocate that if your framework doesn't at least support data, views, and
high level logic as first class entities then it's just a server. You don't
have to use ORM, any particular templating, or Finite State Machines like
Lamson does. You don't even have to settle on only one way to do data, views,
and logic.
You *must* at least support data, views, and logic out of the box so your user
doesn't have to go shopping at "APIs-R-Us" just to use your gear.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/filtering_spam.html 0000644 0000765 0000024 00000024771 11242713200 025443 0 ustar zedshaw staff
LamsonProject: Filtering Spam With Lamson
Lamson supports initial use of the
SpamBayes spam filter library for filtering
spam. What Lamson provides is a set of easy to use decorators that you attach
to your state functions which indicate that you want spam filtered. It also
uses the default SpamBayes configuration files and database formats as you
configure, so if you have an existing SpamBayes setup you should be able to use
it right away.
Using lamson.spam
Lamson gives you a simple decorator to place on any state functions that should
block spam. Typically you do not want spam filtering on your entire
application, since that would prevent legitimate registrations and put too much
burden on your system. It’s better to put spam filtering on the “insider”
parts, and to have confirmation emails on “outsider” pieces.
Instead, what you want is to indicate that your “choke points” are filtering
spam using
lamson.spam.spam_filter
so that when a spam is received they are put into a “spam black hole”.
Here’s an trivial example where the user is in the POSTING state, and you want
everything to work like normal, but if they spam then you flip them into a
SPAMMING state.
The line to look at is obviously the spam_filter line, which tells Lamson that you will:
Use the SpamBayes training database run/spamdb for the detection.
Use the SpamBayes run/.hammierc file for your config (optional and ignored if it is not there).
Use run/spam as the dumping ground for anything classified as spam.
The next_state to transition to if they send a spam message. This is optional, but very helpful.
With this, the spam_filter then wraps your state function, and every message
is fed to SpamBayes. If SpamBayes says it’s spam then Lamson will dump it into
your run/spam and transition to SPAMMING *without running your POSTING
state*.
Once you are in this new SPAMMING state (or any state you like) you can do
whatever you want. You can leave them there, or you can have an external tool
that let’s you un-block someone. Pretty much any spam handling scheme you want
is available.
Since your spam is placed into a queue you can inspect it later and check for
any accidentally miscategorized mail, then use the SpamBayes tools to retrain
for the misdetection.
Lamson only classifies mail that is marked as actual spam by looking at
the 'X-Spambayes-Classification’ header and seeing if it starts with 'spam’.
If it is 'unsure’ or 'ham’ it will let it through.
Effectiveness
I’ve been running a variant of this since the middle of
May 2009 and it works great. The code I run is a custom version that fits the
weirdness of my email setup but the principles are the same. I’m currently
using the above spam filtering, some gray listing, and a few other tricks to
block most of my incoming spam.
With all the spam block measures I’ve managed to cut down my spam to about 2-3
a day out of about 100-200 I receive. The majority of the “spam” that gets
through is actually email that’s classified as “unsure” which I then use to
retrain SpamBayes to make it stronger.
However, that’s my personal server, so in the case of a Lamson application
you’ll want to be careful that your spam blocking activities don’t prevent too
much legitimate use.
Changing What “Spam” Means
You can also change how spam is determined by sub-classing
lamson.spam.spam_filter
and doing your own implementation of the spam method.
Using SpamBayes
An important point about SpamBayes is that it comes with all the command line
tools you need to configure and train your database using a corpus of spam you
might have. All Lamson needs to do is read this database to determine if it is
spam or not.
With mutt, I save the message to “=spam”, which places the spam in Mail/spam
along with all of the others. Then I run this command:
sb_mboxtrain.py -s ~/Mail/spam -d run/spamdb
This goes through the spam mailbox, and any emails that SpamBayes has not
already classified get used for training.
SpamBayes comes with other commands you can read
about on their site (if you can
find it).
Autotraining
Lamson doesn’t support “autotraining” directly, since it’s not clear in each
situation what is obviously spam. In my personal setup I know that any email
not for registered users is obviously spam, so I can autotrain those.
If you want to implement autotraining for a part of your application, then look
at the API for
lamson.spam.Filter
and simply use it in the right state function.
Configuration
Finally, the above sample code is not the best way to configure the spam filter.
It’s better to put the configuration in config/settings.py and simply reference
it from there.
from config.settings import SPAM
@spam_filter(SPAM['db'], SPAM['rc'], SPAM['queue'], next_state=SPAMMING)
def START(message, ...):
# this is the better way to do your config
With that you can then change up the configuration as needed in
your deployments without having to change your code.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/filtering_spam.txt 0000644 0000765 0000024 00000013430 11242713153 025313 0 ustar zedshaw staff Title: Filtering Spam With Lamson
Lamson supports initial use of the
"SpamBayes":http://spambayes.sourceforge.net/ spam filter library for filtering
spam. What Lamson provides is a set of easy to use decorators that you attach
to your state functions which indicate that you want spam filtered. It also
uses the default SpamBayes configuration files and database formats as you
configure, so if you have an existing SpamBayes setup you should be able to use
it right away.
h2. Using lamson.spam
Lamson gives you a simple decorator to place on any state functions that should
block spam. Typically you do *not* want spam filtering on your entire
application, since that would prevent legitimate registrations and put too much
burden on your system. It's better to put spam filtering on the "insider"
parts, and to have confirmation emails on "outsider" pieces.
Instead, what you want is to indicate that your "choke points" are filtering
spam using
"lamson.spam.spam_filter":http://lamsonproject.org/docs/api/lamson.spam.spam_filter-class.html
so that when a spam is received they are put into a "spam black hole".
Here's an trivial example where the user is in the POSTING state, and you want
everything to work like normal, but if they spam then you flip them into a
SPAMMING state.
The line to look at is obviously the @spam_filter@ line, which tells Lamson that you will:
# Use the SpamBayes training database @run/spamdb@ for the detection.
# Use the SpamBayes @run/.hammierc@ file for your config (optional and ignored if it is not there).
# Use @run/spam@ as the dumping ground for anything classified as spam.
# The next_state to transition to if they send a spam message. *This is optional, but very helpful.*
With this, the @spam_filter@ then wraps your state function, and every message
is fed to SpamBayes. If SpamBayes says it's spam then Lamson will dump it into
your @run/spam@ and transition to SPAMMING *without running your POSTING
state*.
Once you are in this new @SPAMMING@ state (or any state you like) you can do
whatever you want. You can leave them there, or you can have an external tool
that let's you un-block someone. Pretty much any spam handling scheme you want
is available.
Since your spam is placed into a queue you can inspect it later and check for
any accidentally miscategorized mail, then use the SpamBayes tools to retrain
for the misdetection.
bq. Lamson only classifies mail that is marked as actual spam by looking at
the 'X-Spambayes-Classification' header and seeing if it starts with 'spam'.
If it is 'unsure' or 'ham' it will let it through.
h2. Effectiveness
"I've":http://zedshaw.com/ been running a variant of this since the middle of
May 2009 and it works great. The code I run is a custom version that fits the
weirdness of my email setup but the principles are the same. I'm currently
using the above spam filtering, some gray listing, and a few other tricks to
block most of my incoming spam.
With all the spam block measures I've managed to cut down my spam to about 2-3
a day out of about 100-200 I receive. The majority of the "spam" that gets
through is actually email that's classified as "unsure" which I then use to
retrain SpamBayes to make it stronger.
However, that's my personal server, so in the case of a Lamson application
you'll want to be careful that your spam blocking activities don't prevent too
much legitimate use.
h2. Changing What "Spam" Means
You can also change how spam is determined by sub-classing
"lamson.spam.spam_filter":http://lamsonproject.org/docs/api/lamson.spam.spam_filter-class.html
and doing your own implementation of the @spam@ method.
h2. Using SpamBayes
An important point about SpamBayes is that it comes with all the command line
tools you need to configure and train your database using a corpus of spam you
might have. All Lamson needs to do is read this database to determine if it is
spam or not.
With mutt, I save the message to "=spam", which places the spam in Mail/spam
along with all of the others. Then I run this command:
sb_mboxtrain.py -s ~/Mail/spam -d run/spamdb
This goes through the spam mailbox, and any emails that SpamBayes has *not*
already classified get used for training.
SpamBayes comes with other commands you can "read
about":http://spambayes.sourceforge.net/docs.html on their site (if you can
find it).
h2. Autotraining
Lamson doesn't support "autotraining" directly, since it's not clear in each
situation what is obviously spam. In my personal setup I know that any email
not for registered users is obviously spam, so I can autotrain those.
If you want to implement autotraining for a part of your application, then look
at the API for
"lamson.spam.Filter":http://lamsonproject.org/docs/api/lamson.spam.Filter-class.html
and simply use it in the right state function.
h2. Configuration
Finally, the above sample code is not the best way to configure the spam filter.
It's better to put the configuration in @config/settings.py@ and simply reference
it from there.
In your @config/settings.py@ put this:
from config.settings import SPAM
@spam_filter(SPAM['db'], SPAM['rc'], SPAM['queue'], next_state=SPAMMING)
def START(message, ...):
# this is the better way to do your config
With that you can then change up the configuration as needed in
your deployments without having to change your code.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/getting_started.html 0000644 0000765 0000024 00000124752 11242713463 025642 0 ustar zedshaw staff
LamsonProject: Getting Started With Lamson
Lamson is designed to work like modern web application frameworks like Django, TurboGears,
ASP.NET, Ruby on Rails, and whatever PHP is using these days. At every design decision
Lamson tries to emulate terminology and features found in these frameworks. This Getting Started
document will help you get through that terminology, get you started running your first
lamson application, and walk you through the code you should read.
In total it should take you about 30 minutes to an hour to complete. If you just want
to try Lamson, at least go through the 30 second introduction given first.
The 30 Second Introduction
If you have Python and easy_install already, then try this out:
$ easy_install lamson
$ lamson gen -project mymailserver
$ cd mymailserver
$ lamson start
$ lamson log
$ nosetests
$ lamson help -for send
$ lamson send -sender me@mydomain.com -to test@test.com \
-subject "My test." -body "Hi there." -port 8823
$ less logs/lamson.log
$ mutt -F muttrc
You now have a working base Lamson setup ready for you to work on with the following installed:
Lamson and all dependencies (Jinja2, nosetests)
Code for your project in mymailserver. Look in app/handlers and config/settings.py.
Two initial tests that verify your server is not an open relay and forwards mail in tests/handlers/open_relay_tests.py.
A “logger” server running on port 8825 that dumps all of its mail into the run/queue maildir.
A config script for mutt (muttrc) that you can use to inspect the run/queue and also send mail using Lamson’s send command.
When you’re in mutt during the above test run, try sending an email. The
included muttrc is configured to use the run/queue as the mail queue, and to
use the lamson sendmail command to deliver the mail. This tricks mutt into
interacting directly with your running Lamson server, so you can test the thing
with a real mail client and see how it will work without having to actually
deploy the server.
Finally, if you wanted to stop all of above you would do:
$ lamson stop -ALL run
Which tells Lamson to stop all processes that have a .pid file in the run directory.
Important Terminology
If you are an old SMTP guru and/or you’ve never written a web application with
a modern web framework, then some of the terminology used in Lamson may seem
confusing. Other terms may just confuse you or scare you because they sound
complicated. I tried my best to make the concepts used in Lamson
understandable and the code that implements them easy to read. In fact, you
could probably read the code to Lamson in an evening and understand how
everything works.
Experience has taught me that nobody reads the code, even if it is
small. Therefore, here are the most important concepts you should know to get
a grasp of Lamson and how it works.
MVC — Model View Controller is a design methodology used in web application frameworks where the data (model), presentation (view), and logic (controller) layers of the application are strictly separated.
FSM — Lamson uses the concept of a Finite State Machine to control how handlers execute. Each time it runs it will perform an action based on what it is send and what it was doing last. FSM in computer science class are overly complex, but in Lamson they are as easy to use as a return statement.
Template — Lamson generates the bodies of its messages using Templates, which are text files that have parts that get replaced with variables you pass in. Templates are converted to their final form with a process called rendering.
Relay — The relay for a Lamson server is where Lamson delivers its messages. Usually the Relay is a smart tougher server that’s not as smart, but very good at delivering mail. Lamson can also be run as a Relay for testing purposes.
Receiver — Lamson typically runs as the Receiver of email. If you are familiar with a web application setup, then Lamson is the inverse. Instead of Lamson runing “behind” an Apache or Nginx server, Lamson runs “in front” of an SMTP server like Postfix. It listens on port 25, handles the mail it should, and forwards the rest to the Relay. This makes Lamson much more of a Proxy or filter server.
Queue — Lamson can also do all of its processing off a queue. In this setup you would have your normal mail server dump all mail to a maildir queue, and then tell Lamson to process messages out of there. This can be combined with the usual Receiver+Relay configuration for processing messages that might take a long time.
Maildir — A standard created for the qmail project with stores mail in a directory such that you can access the mail atomically and store it on a shared disk without conflicts or locking.
Managing Your Server
Your Lamson application is now running inside the Lamson Python server. This is a very simple server based on Python’s
smtpd and asyncore libraries.
If you want to know more about how it operates, take a look at the lamson/server.py file in the source distribution.
You’ll need to use a few Lamson commands to manage the server. You already experienced them in the
30 second introduction, and you can review them all or see them
by using the lamson help command.
Right now you have Lamson running on port 8823 and a “Lamson logger” running on 8825. This means that
your lamson server (port 8823) will forward its messages to the logger (port 8825) thinking it’s your
real relay server. The truth is the logger just logs its messages to logs/logger.log and dumps it into
run/queue so you can inspect the results.
Before we learn how to manage them and what they do, open up the
config/settings.py file and take a look:
Your file probably has some comments telling you what these do, but it’s important to understand
how they work.
First, this file is just plain old Python variables. It is loaded by one of
two other files in your config directory: config/boot.py or
config/testing.py. The config/boot.py file is started whenever you use the
lamson start command and its job is to read the config/settings.py and
start all the services you need, then assign them as variables back to
config.settings so your handlers can get at them. The config/testing.py is
almost the same, except it configures config.settings so that your unit tests
can run without any problems. Typically this means setting the spell checker
and not starting the real server.
Lamson can load any boot script you like, see Deferred Processing To
Queues for an example of using this
to make a queue processor.
The important thing to understand about this setup (where a boot file reads settings.py and
then configures config.settings) that it makes it easy for you to change Lamson’s operations
or start additional services you need and configure them. For the most part you won’t need
to touch boot.py or testing.py until you need to add some new service, change the template
library you want to use, setup a different database ORM, etc. Until then just ignore it.
settings.py Variables
The receiver_config variable is used by the lamson start command to figure out where to listen
for incoming SMTP connections. In a real installation this would be port 25 on your external
IP address. It’s where the internet talks to your server.
The relay_config setting is used by Lamson to figure out where to forward message replies (responses)
for real delivery. Normally this would be a “smart host” running a more established server
like Postfix or Exim to do the grunt work
of delivering to the final recipients.
The handlers variable lists the modules (not files) of the handlers you want to load.
Simply put them here and they’ll be loaded, even the lamson.handlers
modules will work here too.
The router_defaults are for the lamson.routing.Router
class and configure the default
routing regular expressions you plan on using. Typically you’ll at least configure the
host regular expression since that is used in every route and shouldn’t change too often.
Finally, template_config contains the configuration values for the templating system you’ll
be using. Lamson supports either Mako or Jinja2, but defaults to Jinja2.
Looking At config/boot.py
Programmers need to know how everything works before they trust it, so let’s look at the config/boot.py
file and see how these variables are used:
from config import settings
from lamson.routing import Router
from lamson.server import Relay, SMTPReceiver
from lamson.utils import configure_database
from lamson import view
import logging
import logging.config
import jinja2
# configure logging to go to a log file
logging.config.fileConfig("config/logging.conf")
# the relay host to actually send the final message to
settings.relay = Relay(host=settings.relay_config['host'],
port=settings.relay_config['port'], debug=1)
# where to listen for incoming messages
settings.receiver = SMTPReceiver(settings.receiver_config['host'],
settings.receiver_config['port'])
settings.database = configure_database(settings.database_config, also_create=False)
Router.defaults(**settings.router_defaults)
Router.load(settings.handlers)
Router.RELOAD=True
view.LOADER = jinja2.Environment(
loader=jinja2.PackageLoader(settings.template_config['dir'],
settings.template_config['module']))
Don’t be afraid that you see this much Python, you normally wouldn’t touch this file
unless it were to add your own services or to make a new version for a different configuration.
For the most part, you can just edit the config/settings.py and go.
First you’ll see that config/boot.py sets up logging using the config/logging.conf
file, which you can change to reconfigure how you want logs to be created.
Then it starts assigning variables to the config.settings module that it
has imported at the top. This is important because after config.boot runs
your lamson code and handlers will have access to all these services. You
can get directly to the relay, receiver, database and anything else you need
by simply doing:
from config import settings
After that config.boot sets up the settings.relay, settings.receiver,
and settings.database. These three are used heavily in your own Lamson code,
so knowing how to change them if you need to helps you later.
After this we configure the lamson.routing.Router to have your defaults,
load up your handlers, and turn on RELOAD. Setting Router.RELOAD=True
tell the Router to reload all the handlers for each request. Very handy when you
are doing development since you don’t need to reload the server so often.
If you deploy to production, then you’ll want to set this to False since it’s
a performance hit.
Finally, the config.boot does the job os loading the template system you’ll use,
in this case Jinja2. Jinja2 and Mako use the same API so you can configure
Mako here as well, as long as the object assigned to view.LOADER has the same API
it will work.
Developing With Lamson
Now that you’ve received a thorough introduction to how to manage Lamson, and
how it is configured, you can get into actually writing some code for it.
Before you begin, you should know that writing an application for a mail server
can be a pain. The clients and servers that handle SMTP make a large number of
assumptions based on how the world was back in 1975. Everything is on defined ports
with defined command line parameters and the concept of someone pointing their
mail client at a different server arbitrarily just doesn’t exist.
The world of email is not like the web where you just take any old “client” and point
it at any old server and start messing with it.
Lucky for you, Lamson has solved most of these problems and provides you with a
bunch of handy development tools and tricks so you can work with your Lamson server
without having to kill yourself in configuration hell.
Using Mutt
You probably don’t have another SMTP server running, and even if you did, it’d
be a pain to configure it for development purposes. You’d have to setup aliases, new
mail boxes, restart it all the time, and other annoyances.
For development, what we want is our own little private SMTP relay, and since Lamson can
also deliver mail, that is what we get with the command:
$ lamson log
This tells Lamson to run as a “logging server”, which doesn’t actually deliver
any mail. With this one command you have a server running on 8825 that takes every
mail it receives and saves it to the run/queue Maildir and also logs it to
logs/logger.log. It also logs the full protocol chat to logs/lamson.err so
you can inspect it.
Lamson uses Maildir by default since it is the most reliable and fastest mail queue
format available. It could also store mail messages to any queue supported by Python’s
mailbox library. If you were
adventurous you could also use a RDBMS, but that’s just silly.
You also have the file muttrc which is configured to trick mutt into talking to your
running Lamson server, and then read mail out of the run/queue maildir that is filled
in by the lamson log server. Let’s take a look:
set mbox_type=Maildir
set folder="run/queue"
set mask="!^\\.[^.]"
set mbox="run/queue"
set record="+.Sent"
set postponed="+.Drafts"
set spoolfile="run/queue"
set sendmail="/usr/bin/env lamson sendmail -port 8823 -host 127.0.0.1"
Notice that it’s configured sendmail to be “sendmail -port 8823 -host 127.0.0.1”
which is a special lamson sendmail command that knows how to talk to lamson and
read the arguments and input that mutt gives to deliver a mail.
Why does Lamson need its own sendmail? Because you actually have to configure most
mail server’s configuration files to change their ports before their sendmail command
will use a different port. Yes, the average sendmail command line tool assumes that it
is always talking to one and only one server on one and only one port for ever and all
eternity. Without lamson sendmail you wouldn’t be able to send to an arbitrary
server.
With this setup (lamson start ; lamson log ; mutt -F muttrc) you can now
use your mutt client as a test tool for working with your application.
Stopping Lamson
The PID files are stored in the run directory. Here’s a sample
session where I stop all the running servers:
$ ls -l run/*.pid
-rw-r--r-- 1 zedshaw staff 5 May 16 16:41 run/log.pid
-rw-r--r-- 1 zedshaw staff 5 May 16 16:41 run/smtp.pid
$ lamson stop -ALL run
Stopping processes with the following PID files: ['run/log.pid', 'run/smtp.pid']
Attempting to stop lamson at pid 1693
Attempting to stop lamson at pid 1689
You can also pass other options to the stop command to just stop one server. Use
lamson help -for stop to see all the options.
Starting Lamson Again
Hopefully you’ve been paying attention and have figured out how to restart lamson and the
logging server. Just in case, here it is again:
$ lamson start
$ lamson log
You should also look in the logs/lamson.log file to see that it actually started. The
other files in the logs directory contain messages dumped to various output methods (like
Python’s stdout and stderr). Periodically, if the information you want is not in
logs/lamson.log then it is probably in the other files.
You can change your logging configuration by editing the logging line your config/settings.py file.
Other Useful Commands
You should read the available commands documentation to get an
overview, and you can also use lamson help to see them at any time.
send
The first useful command is lamson send, which lets you send mail to SMTP servers (not
just Lamson) and watch the full SMTP protocol chatter. Here’s a sample:
$ lamson send -port 25 -host zedshaw.com -debug 1 \
-sender tester@test.com -to zedshaw@zedshaw.com \
-subject "Hi there" -body "Test body."
send: 'ehlo zedshawscomputer.local\r\n'
reply: '502 Error: command "EHLO" not implemented\r\n'
reply: retcode (502); Msg: Error: command "EHLO" not implemented
send: 'helo zedshawcomputer.local\r\n'
reply: '250 localhost.localdomain\r\n'
reply: retcode (250); Msg: localhost.localdomain
send: 'mail FROM:\r\n'
reply: '250 Ok\r\n'
reply: retcode (250); Msg: Ok
send: 'rcpt TO:\r\n'
reply: '250 Ok\r\n'
reply: retcode (250); Msg: Ok
send: 'data\r\n'
reply: '354 End data with .\r\n'
reply: retcode (354); Msg: End data with .
data: (354, 'End data with .')
send: 'Content-Type: text/plain; charset="us-ascii"\r\nMIME-Version: 1.0\r\nContent-Transfer-Encoding: 7bit\r\nSubject: Hi there\r\nFrom: tester@test.com\r\nTo: zedshaw@zedshaw.com\r\n\r\n.\r\n'
reply: '250 Ok\r\n'
reply: retcode (250); Msg: Ok
data: (250, 'Ok')
send: 'quit\r\n'
reply: '221 Bye\r\n'
reply: retcode (221); Msg: Bye
Using this helps you debug your Lamson server by showing you the exact protocol sent
between you and the server. It is also a useful SMTP server debug command by itself.
When you use the supplied muttrc you’ll be configured to use Lamson’s sendmail
(not *send) command as your delivery command. This lets you use mutt as a complete development
tool with minimal configuration.
queue
The lamson queue command lets you investigate and manipulate the run/queue (or any maildir).
You can pop a message off, get a message by its key, remove a message by its key, count the messages,clear the queue, list keys in the queue. It gives you a lower level view of the queue than
mutt would, and lets you manipulate it behind the scenes.
restart
Lamson does reload the code of your project when it receives a new request (probably too
frequently), but if you change the config/settings.py file then you need to restart.
Easiest way to do that is with the restart command.
Walking Through The Code
You should actually know quite a lot about how to run and mess with Lamson, so you’ll
want to start writing code. Before you do, go check out the API Documentation
and take a look around. This document will guide you through where everything is and how
to write your first handler, but when you start going out on your own you’ll need a good
set of reference material.
At the top level of your newly minted project you have these directories:
app -- Where the application code (handlers, templates, models) lives.
config -- You already saw everything in here.
logs -- Log files get put here.
run -- Stuff that would go in a /var/run like PID files and queues.
tests -- Unit tests for handlers, templates, and models.
Lamson expects all of these directories to be right there, so don’t get
fancy and think you can move them around.
The first place to look is in the app directory, which has this:
app/__init__.py
app/data -- Data you want to keep around goes here.
app/handlers -- Lamson handlers go here.
app/model -- Any type of backend ORM models or other non-handler code.
app/templates -- Email templates.
You don’t technically have to store your data in app/data. You are free
to put it anywhere you want, it’s just convenient for most situations to
have it there.
Your app/model directory could have anything in it from simple modules for
working various Maildir queues, to full blown SQLAlchemy configurations for
your database. The only restriction is that you load them in the modules
yourself (no magic here).
The app/templates directory can have any structure you want, and as you
saw from the config.boot discussion it is just configured into the
Jinja2 configuration as the default. If you have a lot of templates it might
help to have them match your app/handlers layout in some logical way.
That only leaves your app/handlers directory:
app/handlers/__init__.py
app/handlers/sample.py
This is where the world gets started. If you look at your config.settings
you’ll see this line:
handlers = ['app.handlers.sample']
Yep, that’s telling the lamson.routing.Router
to load your app.handlers.sample
module to kick it into gear. It really is as simple as just putting the file in
that directory (in in sub-modules there) and then adding them to the handlers
list.
You can also add handlers from modules outside of your app.handlers:
This installs the handler (lamson.handlers.log) that lamson uses to log every email it receives.
Writing Your Handler
This document is for getting started quickly, so going into the depths of the cool stuff
you can do with Lamson handlers is outside the scope, but if you open the app/handlers/sample.py
file and take a look you’ll how a handler is structured.
Since Lamson is changing so much the contents of the file aren’t included in this document. You’ll have to open it and take a look.
At the top of the file you should see your typical import statements:
import logging
from lamson.routing import route, route_like, stateless
from config.settings import relay, database
from lamson import view
Notice that we include elements from the lamson.routing that are decorators
we use to configure a route. Then you’ll see that we’re getting that settings.relay
and settings.database we configured in the previous sections. Finally we bring
in the lamson.view module directory to make rendering templates into email messages
a lot easier.
Now take a look at the rest of the file and you’ll how a handler is structured:
Each state is a separate function in CAPS. It doesn’t have to be, it just looks better.
Above each state function is a route, route_like, or stateless decorator to configure how lamson.routing.Router uses it.
The route decorator takes a pattern and then regex keyword arguments to fill it in. The words in the pattern string are replaced in the final more complex routing regex by the keyword arguments after. However, if you want to use regex directly you can, route just needs a string that eventually becomes a regex.
A state function changes state by returning the next function to call. You want to go to the RUNNING state, just return RUNNING.
If any state function throws an error it will go into the ERROR state, so if you make a state handler named ERROR it will get called on the next event and can recover.
If you want to run a state on this event rather than wait to have it run on the next, then simple call it and return what it returns. So to have RUNNING go now, just do return RUNNING(message, ...) and it will work.
If a state has the same regex as another state, just use route_like to say that.
If you have a stateless decorator after a route or route_like, then that handler will run for all addresses that match, not just if this handler is in that state.
That is pretty much the entire complexity of how you write a handler. You
setup routes, and return the next step in your conversation as the next
function to run. The lamson.routing.Router then takes each message it receives
and runs it through a processing loop handing it to your states and handlers.
How States Are Run
The best way to see how states are processed is to look at the Router code that does it:
def deliver(self, message):
if self.RELOAD: self.reload()
called_count = 0
for functions, matchkw in self.match(message['to']):
to_call = []
in_state_found = False
for func in functions:
if lamson_setting(func, 'stateless'):
to_call.append(func)
elif not in_state_found and self.in_state(func, message):
to_call.append(func)
in_state_found = True
called_count += len(to_call)
for func in to_call:
if lamson_setting(func, 'nolocking'):
self.call_safely(func, message, matchkw)
else:
with self.call_lock:
self.call_safely(func, message, matchkw)
if called_count == 0:
if self.UNDELIVERABLE_QUEUE:
LOG.debug("Message to %r from %r undeliverable, putting in undeliverable queue.",
message['to'], message['from'])
self.UNDELIVERABLE_QUEUE.push(message)
else:
LOG.debug("Message to %r from %r didn't match any handlers.",
message['to'], message['from'])
What this does is take all the handlers you’ve loaded, and then finds which handlers have a state function that
matches the current message. It then goes through each potential match, and determines which of all the matching
state functions is “in that state”. This means that, even though you have six state functions that answer to
“(list_name)-(action)@(host)” only the one that matches the users current state (say PENDING) will be called next.
As it goes through these functions it also loads up any that are marked “stateless” so they can be called as well.
Finally, it just calls them in order. If the message results in no methods to call, then it will take the message
and tell you this, or put it into an UNDELIVERABLE_QUEUE for you to review it later.
Slight design criticism: Currently the order of these calls is fairly deterministic, but you can’t rely on it.
It’s also not clear if all matching states should run, or just the first. It currently only runs the first match,
but it might be better to run each match from each handler. Suggestions welcome on this.
Debugging Routes
In the old way of doing routing you would edit a large table of “routes” in your config/settings.py file and
then that told Lamson how to run. The problem with this is it was too hard to maintain and too hard to
indicate that different states needed a different route.
The new setup is great because all your routing for each handler module is right there, and it’s easy to see
what will cause a particular state function to go off.
What sucks about the new setup is that you can’t find out what all the routes are doing globally in one
place. That’s where lamson routes comes in. Simply run that command and you’ll get a debug dump of
all the full routing regex and the functions and modules they belong to:
This is telling you which regex is matched first, then what those regex are mapped to. This is very handy as you can copy-paste
that regex right into a python shell and then play with it to see if it would match what you want.
You can also pass in an email address to the -test option and it will tell you what routes would match
and which functions that will call:
osb $ lamson routes -test test.blog@oneshotblog.com
2009-06-07 02:33:31,678 - root - INFO - Database configured to use sqlite:///app/data/main.db URL.
Routing ORDER: [... lots of regex here ...]
Routing TABLE:
---
... each regex and what state functions it maps ..
---
'^post-confirm-(?P[a-z0-9]+)@(?Poneshotblog\\.com)$': app.handlers.post.CONFIRMING
---
TEST address 'test.blog@oneshotblog.com' matches:
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.index.POSTING
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.post.START
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.post.POSTING
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
osb $
If you’re working with Lamson this is incredibly helpful, because it tells you what
routes you have, what functions they call, and then it’ll take an email address and
tell you all the routes that match it.
THREADING!
Lamson takes a lighter approach to how it runs. It assumes that most of the
time you want lamson to keep itself sane with minimal locking, and that you
want each of your state functions to run in a thread lock that prevents others
from stepping on your operations. In 95% of the cases, this is what you want.
To accomplish this, Lamson’s router will acquire an internal lock for
operations that change its state, and a separate lock before it calls each
state function. Since multiple state functions run inside each thread, but one
thread handles each message, you’ll get multiple processing, but each state
won’t step on other states in the system.
However, it’s those 5% of the times that will kill your application, and if you
know what you’re doing, you should be able to turn this off. In order to tell
the Router not to lock your state function, simply decorate it with
nolocking
and Lamson will skip the locking and just run your state raw. This means that
other threads will run potentially stepping on your execution, so you must do
your own locking.
Now, don’t think that slapping a
nolocking
on your state functions is some magic cure for performance issues. You only
ever want to do this if you really know your stuff, and you know how to make
that operation faster with better controlled locking.
The reality is, if you have an operation that takes so long it blocks
everything else, then you are doing it wrong by trying to do it all in your
state function. You should change your design so that this handler drops the
message into a
lamson.queue.Queue
and that another Lamson server reads messages out of that to do the long
running processing.
Using queues and separate Lamson servers you can solve most of your processing
issues without a lot of thread juggling and process locking. In fact, since
Lamson uses maildir queues by default you can even spread these processors out
to multiple machines reading off a shared disk and everything will be just
fine.
But, since programmers will always want to just try turning off the locking,
Lamson supports the nolocking decorator. Use with care.
What’s In A Unit Test
Writing unit tests is way outside the scope of this document, but you should read up on using nosetests, testunit, and
you should look at lamson.testing for a bunch of helper functions. Also look in the generated tests directory
to see some examples.
Spell Checking Your Email Templates
Another big help is that Lamson has support for PyEnchant so you
can spell check your templates. You can use lamson.testing.spelling function in your unit tests.
Installing PyEnchant is kind of a pain, but the trick is to get the dictionary you want and put it in your
~/.enchant/myspell directory. You’ll also want to open the config/testing.py file and uncomment the
lines at the bottom that tell PyEnchant where to find the enchant so (dylib).
PyEnchant is kind of hard to use, so if you have suggestions on a better Python spell checking lib for
unit tests please let me know.
Spam Filtering For Free
Lamson comes with the
lamson.spam module
which supports SpamBayes spam filtering
system.
Read the document on Filtering Spam With Lamson to get
a full set of instructions on using the spam filtering features.
Other Examples
Next you’ll want to sink your teeth in a bigger example. Go grab the source distribution .tar.gz and
extract it so you can get at the examples:
$ tar -xzvf lamson-VERSION.tar.gz
$ cd lamson-VERSION
$ cd examples/osb
You are now in the osb example that is running on
oneshotblog.com. Using what you’ve learned so far
you can start reviewing the code and finding out how a working example
operates.
Getting Help
As you work through this documentation, send your questions to me and I’ll try to help
you. You can also join the lamson@librelist.com mailing list and get help
from other Lamson users.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/getting_started.txt 0000644 0000765 0000024 00000104617 11242713461 025511 0 ustar zedshaw staff From: Zed
Title: Getting Started With Lamson
Lamson is designed to work like modern web application frameworks like Django, TurboGears,
ASP.NET, Ruby on Rails, and whatever PHP is using these days. At every design decision
Lamson tries to emulate terminology and features found in these frameworks. This Getting Started
document will help you get through that terminology, get you started running your first
lamson application, and walk you through the code you should read.
In total it should take you about 30 minutes to an hour to complete. If you just want
to try Lamson, at least go through the 30 *second* introduction given first.
h2. The 30 Second Introduction
If you have Python and "easy_install":http://peak.telecommunity.com/DevCenter/EasyInstall already, then try this out:
$ easy_install lamson
$ lamson gen -project mymailserver
$ cd mymailserver
$ lamson start
$ lamson log
$ nosetests
$ lamson help -for send
$ lamson send -sender me@mydomain.com -to test@test.com \
-subject "My test." -body "Hi there." -port 8823
$ less logs/lamson.log
$ mutt -F muttrc
You now have a working base Lamson setup ready for you to work on with the following installed:
* Lamson and all dependencies (Jinja2, nosetests)
* Code for your project in mymailserver. Look in app/handlers and config/settings.py.
* Two initial tests that verify your server is not an open relay and forwards mail in tests/handlers/open_relay_tests.py.
* A "logger" server running on port 8825 that dumps all of its mail into the run/queue maildir.
* A config script for mutt (muttrc) that you can use to inspect the run/queue *and* also send mail using Lamson's *send* command.
When you're in mutt during the above test run, try sending an email. The
included muttrc is configured to use the run/queue as the mail queue, and to
use the @lamson sendmail@ command to deliver the mail. This tricks mutt into
interacting directly with your running Lamson server, so you can test the thing
with a real mail client and see how it will work without having to actually
deploy the server.
Finally, if you wanted to stop all of above you would do:
$ lamson stop -ALL run
Which tells Lamson to stop all processes that have a .pid file in the @run@ directory.
h2. Important Terminology
If you are an old SMTP guru and/or you've never written a web application with
a modern web framework, then some of the terminology used in Lamson may seem
confusing. Other terms may just confuse you or scare you because they sound
complicated. I tried my best to make the concepts used in Lamson
understandable and the code that implements them easy to read. In fact, you
could probably read the code to Lamson in an evening and understand how
everything works.
Experience has taught me that nobody reads the code, even if it is
small. Therefore, here are the most important concepts you should know to get
a grasp of Lamson and how it works.
* MVC(Model View Controller) -- Model View Controller is a design methodology used in web application frameworks where the data (model), presentation (view), and logic (controller) layers of the application are strictly separated.
* FSM(Finite State Machine) -- Lamson uses the concept of a Finite State Machine to control how handlers execute. Each time it runs it will perform an action based on what it is send *and* what it was doing last. FSM in computer science class are overly complex, but in Lamson they are as easy to use as a @return@ statement.
* Template -- Lamson generates the bodies of its messages using Templates, which are text files that have parts that get replaced with variables you pass in. Templates are converted to their final form with a process called *rendering*.
* Relay -- The *relay* for a Lamson server is where Lamson delivers its messages. Usually the Relay is a smart tougher server that's not as smart, but very good at delivering mail. Lamson can also be run as a Relay for testing purposes.
* Receiver -- Lamson typically runs as the Receiver of email. If you are familiar with a web application setup, then Lamson is the inverse. Instead of Lamson runing "behind" an Apache or Nginx server, Lamson runs "in front" of an SMTP server like Postfix. It listens on port 25, handles the mail it should, and forwards the rest to the Relay. This makes Lamson much more of a Proxy or filter server.
* Queue -- Lamson can also do all of its processing off a queue. In this setup you would have your normal mail server dump all mail to a maildir queue, and then tell Lamson to process messages out of there. This can be combined with the usual Receiver+Relay configuration for processing messages that might take a long time.
* Maildir -- A standard created for the qmail project with stores mail in a directory such that you can access the mail atomically and store it on a shared disk without conflicts or locking.
h2. Managing Your Server
Your Lamson application is now running inside the Lamson Python server. This is a very simple server based on Python's
"smtpd":http://docs.python.org/library/smtpd.html and "asyncore":http://docs.python.org/library/asyncore.html libraries.
bq. If you want to know more about how it operates, take a look at the @lamson/server.py@ file in the source distribution.
You'll need to use a few Lamson commands to manage the server. You already experienced them in the
30 second introduction, and you can review "them all":/docs/lamson_commands.html or see them
by using the @lamson help@ command.
Right now you have Lamson running on port 8823 and a "Lamson logger" running on 8825. This means that
your lamson server (port 8823) will forward its messages to the logger (port 8825) thinking it's your
real relay server. The truth is the logger just logs its messages to logs/logger.log and dumps it into
run/queue so you can inspect the results.
Before we learn how to manage them and what they do, open up the
@config/settings.py@ file and take a look:
Your file probably has some comments telling you what these do, but it's important to understand
how they work.
First, this file is just plain old Python variables. It is loaded by one of
two other files in your config directory: @config/boot.py@ or
@config/testing.py@. The @config/boot.py@ file is started whenever you use the
@lamson start@ command and its job is to read the @config/settings.py@ and
start all the services you need, then assign them as variables back to
@config.settings@ so your handlers can get at them. The @config/testing.py@ is
almost the same, except it configures @config.settings@ so that your unit tests
can run without any problems. Typically this means setting the spell checker
and *not* starting the real server.
bq. Lamson can load any boot script you like, see "Deferred Processing To
Queues":/docs/deferred_processing_to_queues.html for an example of using this
to make a queue processor.
The important thing to understand about this setup (where a boot file reads settings.py and
then configures @config.settings@) that it makes it easy for you to change Lamson's operations
or start additional services you need and configure them. For the most part you won't need
to touch @boot.py@ or @testing.py@ until you need to add some new service, change the template
library you want to use, setup a different database ORM, etc. Until then just ignore it.
h2. settings.py Variables
The @receiver_config@ variable is used by the _lamson start_ command to figure out where to listen
for incoming SMTP connections. In a real installation this would be port *25* on your external
IP address. It's where the internet talks to your server.
The @relay_config@ setting is used by Lamson to figure out where to forward message replies (responses)
for real delivery. Normally this would be a "smart host" running a more established server
like "Postfix":http://www.postfix.org/ or "Exim":http://www.exim.org/ to do the grunt work
of delivering to the final recipients.
The @handlers@ variable lists the modules (not files) of the handlers you want to load.
Simply put them here and they'll be loaded, even the "lamson.handlers":http://lamsonproject.org/docs/api/lamson.handlers-module.html
modules will work here too.
The @router_defaults@ are for the "lamson.routing.Router":http://lamsonproject.org/docs/api/lamson.routing.RoutingBase-class.html
class and configure the default
routing regular expressions you plan on using. Typically you'll at least configure the
@host@ regular expression since that is used in every route and shouldn't change too often.
Finally, @template_config@ contains the configuration values for the templating system you'll
be using. Lamson supports either Mako or Jinja2, but defaults to Jinja2.
h2. Looking At config/boot.py
Programmers need to know how everything works before they trust it, so let's look at the _config/boot.py_
file and see how these variables are used:
from config import settings
from lamson.routing import Router
from lamson.server import Relay, SMTPReceiver
from lamson.utils import configure_database
from lamson import view
import logging
import logging.config
import jinja2
# configure logging to go to a log file
logging.config.fileConfig("config/logging.conf")
# the relay host to actually send the final message to
settings.relay = Relay(host=settings.relay_config['host'],
port=settings.relay_config['port'], debug=1)
# where to listen for incoming messages
settings.receiver = SMTPReceiver(settings.receiver_config['host'],
settings.receiver_config['port'])
settings.database = configure_database(settings.database_config, also_create=False)
Router.defaults(**settings.router_defaults)
Router.load(settings.handlers)
Router.RELOAD=True
view.LOADER = jinja2.Environment(
loader=jinja2.PackageLoader(settings.template_config['dir'],
settings.template_config['module']))
bq. Don't be afraid that you see this much Python, you normally wouldn't touch this file
unless it were to add your own services or to make a new version for a different configuration.
For the most part, you can just edit the @config/settings.py@ and go.
First you'll see that @config/boot.py@ sets up logging using the @config/logging.conf@
file, which you can change to reconfigure how you want logs to be created.
Then it starts assigning variables to the config.settings module that it
has imported at the top. This is important because after @config.boot@ runs
your lamson code and handlers will have access to all these services. You
can get directly to the relay, receiver, database and anything else you need
by simply doing:
from config import settings
After that @config.boot@ sets up the @settings.relay@, @settings.receiver@,
and @settings.database@. These three are used heavily in your own Lamson code,
so knowing how to change them if you need to helps you later.
After this we configure the @lamson.routing.Router@ to have your defaults,
load up your handlers, and turn on RELOAD. Setting @Router.RELOAD=True@
tell the Router to reload all the handlers for each request. Very handy when you
are doing development since you don't need to reload the server so often.
bq. If you deploy to production, then you'll want to set this to False since it's
a performance hit.
Finally, the @config.boot@ does the job os loading the template system you'll use,
in this case Jinja2. Jinja2 and Mako use the same API so you can configure
Mako here as well, as long as the object assigned to view.LOADER has the same API
it will work.
h1. Developing With Lamson
Now that you've received a thorough introduction to how to manage Lamson, and
how it is configured, you can get into actually writing some code for it.
Before you begin, you should know that writing an application for a mail server
can be a pain. The clients and servers that handle SMTP make a large number of
assumptions based on how the world was back in 1975. Everything is on defined ports
with defined command line parameters and the concept of someone pointing their
mail client at a different server arbitrarily just doesn't exist.
The world of email is not like the web where you just take any old "client" and point
it at any old server and start messing with it.
Lucky for you, Lamson has solved most of these problems and provides you with a
bunch of handy development tools and tricks so you can work with your Lamson server
without having to kill yourself in configuration hell.
h2. Using Mutt
You probably don't have another SMTP server running, and even if you did, it'd
be a pain to configure it for development purposes. You'd have to setup aliases, new
mail boxes, restart it all the time, and other annoyances.
For development, what we want is our own little private SMTP relay, and since Lamson can
also deliver mail, that is what we get with the command:
$ lamson log
This tells Lamson to run as a "logging server", which doesn't actually deliver
any mail. With this one command you have a server running on 8825 that takes every
mail it receives and saves it to the @run/queue@ Maildir and also logs it to
@logs/logger.log@. It also logs the full protocol chat to @logs/lamson.err@ so
you can inspect it.
bq. Lamson uses Maildir by default since it is the most reliable and fastest mail queue
format available. It could also store mail messages to any queue supported by Python's
"mailbox":http://docs.python.org/library/mailbox.html library. If you were
adventurous you could also use a RDBMS, but that's just silly.
You also have the file @muttrc@ which is configured to trick mutt into talking to *your*
running Lamson server, and then read mail out of the @run/queue@ maildir that is filled
in by the @lamson log@ server. Let's take a look:
set mbox_type=Maildir
set folder="run/queue"
set mask="!^\\.[^.]"
set mbox="run/queue"
set record="+.Sent"
set postponed="+.Drafts"
set spoolfile="run/queue"
set sendmail="/usr/bin/env lamson sendmail -port 8823 -host 127.0.0.1"
Notice that it's configured sendmail to be "sendmail -port 8823 -host 127.0.0.1"
which is a special @lamson sendmail@ command that knows how to talk to lamson and
read the arguments and input that mutt gives to deliver a mail.
bq. Why does Lamson need its own sendmail? Because you actually have to configure most
mail server's configuration files to change their ports before their *sendmail command*
will use a different port. Yes, the average sendmail command line tool assumes that it
is always talking to one and only one server on one and only one port for ever and all
eternity. Without @lamson sendmail@ you wouldn't be able to send to an arbitrary
server.
With this setup (@lamson start@ ; @lamson log@ ; @mutt -F muttrc@) you can now
use your mutt client as a test tool for working with your application.
h2. Stopping Lamson
The PID(Process ID) files are stored in the @run@ directory. Here's a sample
session where I stop all the running servers:
$ ls -l run/*.pid
-rw-r--r-- 1 zedshaw staff 5 May 16 16:41 run/log.pid
-rw-r--r-- 1 zedshaw staff 5 May 16 16:41 run/smtp.pid
$ lamson stop -ALL run
Stopping processes with the following PID files: ['run/log.pid', 'run/smtp.pid']
Attempting to stop lamson at pid 1693
Attempting to stop lamson at pid 1689
You can also pass other options to the stop command to just stop one server. Use
_lamson help -for stop_ to see all the options.
h2. Starting Lamson Again
Hopefully you've been paying attention and have figured out how to restart lamson and the
logging server. Just in case, here it is again:
$ lamson start
$ lamson log
You should also look in the logs/lamson.log file to see that it actually started. The
other files in the logs directory contain messages dumped to various output methods (like
Python's stdout and stderr). Periodically, if the information you want is not in
logs/lamson.log then it is probably in the other files.
bq. You can change your logging configuration by editing the logging line your config/settings.py file.
h2. Other Useful Commands
You should read the "available commands":/docs/lamson_commands.html documentation to get an
overview, and you can also use _lamson help_ to see them at any time.
h2. send
The first useful command is _lamson send_, which lets you send mail to SMTP servers (not
just Lamson) and watch the full SMTP protocol chatter. Here's a sample:
$ lamson send -port 25 -host zedshaw.com -debug 1 \
-sender tester@test.com -to zedshaw@zedshaw.com \
-subject "Hi there" -body "Test body."
send: 'ehlo zedshawscomputer.local\r\n'
reply: '502 Error: command "EHLO" not implemented\r\n'
reply: retcode (502); Msg: Error: command "EHLO" not implemented
send: 'helo zedshawcomputer.local\r\n'
reply: '250 localhost.localdomain\r\n'
reply: retcode (250); Msg: localhost.localdomain
send: 'mail FROM:\r\n'
reply: '250 Ok\r\n'
reply: retcode (250); Msg: Ok
send: 'rcpt TO:\r\n'
reply: '250 Ok\r\n'
reply: retcode (250); Msg: Ok
send: 'data\r\n'
reply: '354 End data with .\r\n'
reply: retcode (354); Msg: End data with .
data: (354, 'End data with .')
send: 'Content-Type: text/plain; charset="us-ascii"\r\nMIME-Version: 1.0\r\nContent-Transfer-Encoding: 7bit\r\nSubject: Hi there\r\nFrom: tester@test.com\r\nTo: zedshaw@zedshaw.com\r\n\r\n.\r\n'
reply: '250 Ok\r\n'
reply: retcode (250); Msg: Ok
data: (250, 'Ok')
send: 'quit\r\n'
reply: '221 Bye\r\n'
reply: retcode (221); Msg: Bye
Using this helps you debug your Lamson server by showing you the exact protocol sent
between you and the server. It is also a useful SMTP server debug command by itself.
bq. When you use the supplied muttrc you'll be configured to use Lamson's sendmail
(not *send) command as your delivery command. This lets you use mutt as a complete development
tool with minimal configuration.
h2. queue
The _lamson queue_ command lets you investigate and manipulate the run/queue (or any maildir).
You can pop a message off, get a message by its key, remove a message by its key, count the messages,clear the queue, list keys in the queue. It gives you a lower level view of the queue than
mutt would, and lets you manipulate it behind the scenes.
h2. restart
Lamson does reload the code of your project when it receives a new request (probably too
frequently), but if you change the @config/settings.py@ file then you need to restart.
Easiest way to do that is with the restart command.
h2. Walking Through The Code
You should actually know quite a lot about how to run and mess with Lamson, so you'll
want to start writing code. Before you do, go check out the "API Documentation":/docs/api/
and take a look around. This document will guide you through where everything is and how
to write your first handler, but when you start going out on your own you'll need a good
set of reference material.
At the top level of your newly minted project you have these directories:
app -- Where the application code (handlers, templates, models) lives.
config -- You already saw everything in here.
logs -- Log files get put here.
run -- Stuff that would go in a /var/run like PID files and queues.
tests -- Unit tests for handlers, templates, and models.
Lamson expects all of these directories to be right there, so don't get
fancy and think you can move them around.
The first place to look is in the app directory, which has this:
app/__init__.py
app/data -- Data you want to keep around goes here.
app/handlers -- Lamson handlers go here.
app/model -- Any type of backend ORM models or other non-handler code.
app/templates -- Email templates.
You don't technically *have* to store your data in app/data. You are free
to put it anywhere you want, it's just convenient for most situations to
have it there.
Your @app/model@ directory could have anything in it from simple modules for
working various Maildir queues, to full blown SQLAlchemy configurations for
your database. The only restriction is that you load them in the modules
yourself (no magic here).
The @app/templates@ directory can have any structure you want, and as you
saw from the @config.boot@ discussion it is just configured into the
Jinja2 configuration as the default. If you have a lot of templates it might
help to have them match your @app/handlers@ layout in some logical way.
That only leaves your @app/handlers@ directory:
app/handlers/__init__.py
app/handlers/sample.py
This is where the world gets started. If you look at your @config.settings@
you'll see this line:
handlers = ['app.handlers.sample']
Yep, that's telling the "lamson.routing.Router":http://lamsonproject.org/docs/api/lamson.routing.RoutingBase-class.html
to load your @app.handlers.sample@
module to kick it into gear. It really is as simple as just putting the file in
that directory (in in sub-modules there) and then adding them to the handlers
list.
You can also add handlers from modules outside of your @app.handlers@:
This installs the handler ("lamson.handlers.log":http://lamsonproject.org/docs/api/lamson.handlers.log-module.html) that lamson uses to log every email it receives.
h2. Writing Your Handler
This document is for getting started quickly, so going into the depths of the cool stuff
you can do with Lamson handlers is outside the scope, but if you open the _app/handlers/sample.py_
file and take a look you'll how a handler is structured.
bq. Since Lamson is changing so much the contents of the file aren't included in this document. You'll have to open it and take a look.
At the top of the file you should see your typical import statements:
import logging
from lamson.routing import route, route_like, stateless
from config.settings import relay, database
from lamson import view
Notice that we include elements from the @lamson.routing@ that are decorators
we use to configure a route. Then you'll see that we're getting that @settings.relay@
and @settings.database@ we configured in the previous sections. Finally we bring
in the @lamson.view@ module directory to make rendering templates into email messages
a lot easier.
Now take a look at the rest of the file and you'll how a handler is structured:
# Each state is a separate function in CAPS. It doesn't have to be, it just looks better.
# Above each state function is a "route":http://lamsonproject.org/docs/api/lamson.routing.route-class.html, "route_like":http://lamsonproject.org/docs/api/lamson.routing.route_like-class.html, or "stateless":http://lamsonproject.org/docs/api/lamson.routing-module.html#stateless decorator to configure how @lamson.routing.Router@ uses it.
# The "route":http://lamsonproject.org/docs/api/lamson.routing.route-class.html decorator takes a pattern and then regex keyword arguments to fill it in. The words in the pattern string are replaced in the final more complex routing regex by the keyword arguments after. However, *if you want to use regex directly you can*, "route":http://lamsonproject.org/docs/api/lamson.routing.route-class.html just needs a string that eventually becomes a regex.
# A state function changes state by returning the next function to call. You want to go to the RUNNING state, just @return RUNNING@.
# If any state function throws an error it will go into the @ERROR@ state, so if you make a state handler named ERROR it will get called on the next event and can recover.
# If you want to run a state on this event rather than wait to have it run on the next, then simple call it and return what it returns. So to have RUNNING go now, just do @return RUNNING(message, ...)@ and it will work.
# If a state has the same regex as another state, just use "route_like":http://lamsonproject.org/docs/api/lamson.routing.route_like-class.html to say that.
# If you have a "stateless":http://lamsonproject.org/docs/api/lamson.routing-module.html#stateless decorator after a "route":http://lamsonproject.org/docs/api/lamson.routing.route-class.html or "route_like":http://lamsonproject.org/docs/api/lamson.routing.route_like-class.html, then that handler will run for *all* addresses that match, not just if this handler is in that state.
That is pretty much the entire complexity of how you write a handler. You
setup routes, and return the next step in your conversation as the next
function to run. The @lamson.routing.Router@ then takes each message it receives
and runs it through a processing loop handing it to your states and handlers.
h2. How States Are Run
The best way to see how states are processed is to look at the "Router":http://lamsonproject.org/docs/api/lamson.routing.RoutingBase-class.html code that does it:
def deliver(self, message):
if self.RELOAD: self.reload()
called_count = 0
for functions, matchkw in self.match(message['to']):
to_call = []
in_state_found = False
for func in functions:
if lamson_setting(func, 'stateless'):
to_call.append(func)
elif not in_state_found and self.in_state(func, message):
to_call.append(func)
in_state_found = True
called_count += len(to_call)
for func in to_call:
if lamson_setting(func, 'nolocking'):
self.call_safely(func, message, matchkw)
else:
with self.call_lock:
self.call_safely(func, message, matchkw)
if called_count == 0:
if self.UNDELIVERABLE_QUEUE:
LOG.debug("Message to %r from %r undeliverable, putting in undeliverable queue.",
message['to'], message['from'])
self.UNDELIVERABLE_QUEUE.push(message)
else:
LOG.debug("Message to %r from %r didn't match any handlers.",
message['to'], message['from'])
What this does is take all the handlers you've loaded, and then finds which handlers have a state function that
matches the current message. It then goes through each potential match, and determines which of all the matching
state functions is "in that state". This means that, even though you have six state functions that answer to
"(list_name)-(action)@(host)" only the one that matches the users current state (say PENDING) will be called next.
As it goes through these functions it also loads up any that are marked "stateless" so they can be called as well.
Finally, it just calls them in order. If the message results in no methods to call, then it will take the message
and tell you this, or put it into an @UNDELIVERABLE_QUEUE@ for you to review it later.
bq. Slight design criticism: Currently the order of these calls is fairly deterministic, but you can't rely on it.
It's also not clear if *all* matching states should run, or just the first. It currently only runs the first match,
but it might be better to run each match from each handler. Suggestions welcome on this.
h2. Debugging Routes
In the old way of doing routing you would edit a large table of "routes" in your @config/settings.py@ file and
then that told Lamson how to run. The problem with this is it was too hard to maintain and too hard to
indicate that different states needed a different route.
The new setup is great because all your routing for each handler module is right there, and it's easy to see
what will cause a particular state function to go off.
What sucks about the new setup is that you can't find out what all the routes are doing *globally* in one
place. That's where @lamson routes@ comes in. Simply run that command and you'll get a debug dump of
all the full routing regex and the functions and modules they belong to:
This is telling you which regex is matched first, then what those regex are mapped to. This is very handy as you can copy-paste
that regex right into a python shell and then play with it to see if it would match what you want.
You can also pass in an email address to the @-test@ option and it will tell you what routes would match
and which functions that will call:
osb $ lamson routes -test test.blog@oneshotblog.com
2009-06-07 02:33:31,678 - root - INFO - Database configured to use sqlite:///app/data/main.db URL.
Routing ORDER: [... lots of regex here ...]
Routing TABLE:
---
... each regex and what state functions it maps ..
---
'^post-confirm-(?P[a-z0-9]+)@(?Poneshotblog\\.com)$': app.handlers.post.CONFIRMING
---
TEST address 'test.blog@oneshotblog.com' matches:
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.index.POSTING
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.post.START
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
'^(?P[a-zA-Z0-9][a-zA-Z0-9.]+)@(?Poneshotblog\\.com)$' app.handlers.post.POSTING
- {'host': 'oneshotblog.com', 'post_name': 'test.blog'}
osb $
If you're working with Lamson this is incredibly helpful, because it tells you what
routes you have, what functions they call, and then it'll take an email address and
tell you all the routes that match it.
h2. THREADING!
Lamson takes a lighter approach to how it runs. It assumes that most of the
time you want lamson to keep itself sane with minimal locking, and that you
want each of your state functions to run in a thread lock that prevents others
from stepping on your operations. In 95% of the cases, this is what you want.
To accomplish this, Lamson's router will acquire an internal lock for
operations that change its state, and a separate lock before it calls each
state function. Since multiple state functions run inside each thread, but one
thread handles each message, you'll get multiple processing, but each state
won't step on other states in the system.
However, it's those 5% of the times that will kill your application, and if you
know what you're doing, you should be able to turn this off. In order to tell
the Router *not* to lock your state function, simply decorate it with
"nolocking":http://lamsonproject.org/docs/api/lamson.routing-module.html#nolocking
and Lamson will skip the locking and just run your state raw. This means that
other threads will run potentially stepping on your execution, so you *must* do
your own locking.
Now, don't think that slapping a
"nolocking":http://lamsonproject.org/docs/api/lamson.routing-module.html#nolocking
on your state functions is some magic cure for performance issues. You only
ever want to do this if you *really* know your stuff, and you know how to make
that operation faster with better controlled locking.
The reality is, if you have an operation that takes so long it blocks
everything else, then you are doing it wrong by trying to do it all in your
state function. You should change your design so that this handler drops the
message into a
"lamson.queue.Queue":http://lamsonproject.org/docs/api/lamson.queue.Queue-class.html
and that *another* Lamson server reads messages out of that to do the long
running processing.
Using queues and separate Lamson servers you can solve most of your processing
issues without a lot of thread juggling and process locking. In fact, since
Lamson uses maildir queues by default you can even spread these processors out
to multiple machines reading off a shared disk and everything will be just
fine.
But, since programmers will always want to just try turning off the locking,
Lamson supports the @nolocking@ decorator. Use with care.
h2. What's In A Unit Test
Writing unit tests is way outside the scope of this document, but you should read up on using nosetests, testunit, and
you should look at "lamson.testing":http://lamsonproject.org/docs/api/lamson.testing-module.html for a bunch of helper functions. Also look in the generated @tests@ directory
to see some examples.
h2. Spell Checking Your Email Templates
Another big help is that Lamson has support for "PyEnchant":http://www.rfk.id.au/software/pyenchant/ so you
can spell check your templates. You can use "lamson.testing.spelling":http://lamsonproject.org/docs/api/lamson.testing-module.html#spelling function in your unit tests.
Installing PyEnchant is kind of a pain, but the trick is to get the dictionary you want and put it in your
@~/.enchant/myspell@ directory. You'll also want to open the @config/testing.py@ file and uncomment the
lines at the bottom that tell PyEnchant where to find the enchant so (dylib).
PyEnchant is kind of hard to use, so if you have suggestions on a better Python spell checking lib for
unit tests please let "me know.":/contact.html
h2. Spam Filtering For Free
Lamson comes with the
"lamson.spam":http://lamsonproject.org/docs/api/lamson.spam-module.html module
which supports "SpamBayes":http://spambayes.sourceforge.net/ spam filtering
system.
Read the document on "Filtering Spam With Lamson":/docs/filtering_spam.html to get
a full set of instructions on using the spam filtering features.
h2. Other Examples
Next you'll want to sink your teeth in a bigger example. Go grab "the source distribution .tar.gz":/releases/ and
extract it so you can get at the examples:
$ tar -xzvf lamson-VERSION.tar.gz
$ cd lamson-VERSION
$ cd examples/osb
You are now in the osb example that is running on
"oneshotblog.com":http://oneshotblog.com/. Using what you've learned so far
you can start reviewing the code and finding out how a working example
operates.
h2. Getting Help
As you work through this documentation, send your questions "to me":/contact.html and I'll try to help
you. You can also join the "lamson@librelist.com mailing list":mailto:lamson@librelist.com and get help
from other Lamson users.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/hooking_into_django.html 0000644 0000765 0000024 00000015467 11243575424 026472 0 ustar zedshaw staff
LamsonProject: Hooking Lamson Into Django
This is a short document because using DjangoORM is very simple from Lamson. The trick is
to fake out Django so that when you import the Django model into Lamson, the Django model
knows where it’s living and will operate. We’ll go through an “integration” step
by step.
Step 1: Make Your Django App Work
There’s no point in trying to import an ORM that doesn’t work. So get it working,
maybe write some tests.
An important thing is that you should be able to run python manage.py shell and import
your model without problems.
Step 2: Fake Out Django
You then have to put an environment variable in your Lamson config/settings.py file before
you load any Django Models:
The only thing you’ll have to contend with is where code you need to use these models
lives. Since it’s all Python, you can just import what you need, but my recommendation
is to focus most of your model code into your Django application, and then only put
a small amount into Lamson.
For more information, look in the Lamson source releases to see how
Django is used in the examples/librelist code.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/hooking_into_django.txt 0000644 0000765 0000024 00000005235 11243575422 026333 0 ustar zedshaw staff Title: Hooking Lamson Into Django
This is a short document because using "Django":http://www.djangoproject.com/ ORM is very simple from Lamson. The trick is
to fake out Django so that when you import the Django model into Lamson, the Django model
knows where it's living and will operate. We'll go through an "integration" step
by step.
h1. Step 1: Make Your Django App Work
There's no point in trying to import an ORM that doesn't work. So get it working,
maybe write some tests.
An important thing is that you should be able to run @python manage.py shell@ and import
your model without problems.
h2. Step 2: Fake Out Django
You then have to put an environment variable in your Lamson @config/settings.py@ file before
you load any Django Models:
This is from the "librelist.com":http://librelist.com/ examples, where the Django
models are in @webapp@ so we our settings module from Lamson perspective is @webapp.settings@.
h2. Step 3: Use The Django Models
After that, you can just import your models however you want. Here's an example of
Librelist using a Django Model to store confirmations:
h2. Well, That Wasn't Too Hard
The only thing you'll have to contend with is where code you need to use these models
lives. Since it's all Python, you can just import what you need, but my recommendation
is to focus most of your model code into your Django application, and then only put
a small amount into Lamson.
For more information, look in the "Lamson source releases":/releases/ to see how
Django is used in the @examples/librelist@ code.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/html_email_generation.html 0000644 0000765 0000024 00000026156 11243706471 027002 0 ustar zedshaw staff
LamsonProject: HTML Email Generation
HTML Email is apparently the killer feature for everyone who uses email.
The first, second, and third question I get asked when I tell people
about Lamson is, “Does it do HTML email?!” Yes it does, and hopefully
in a very nice clean way that’ll make blasting out all that great
marketing material your users love easier than ever.
The fourth question is whether Lamson handles bounces. Yes it does.
A Few Tips About HTML Email
First, you must educate your marketing people that your user’s Inbox is not a TV.
I actually run myinboxisnota.tv just to make a point that people who
do marketing have the wrong idea of people’s Inbox experience. It’s your job as the hacker to
educate them and make sure that they know users actually mostly hate HTML email, and that sending
them an HTML email is nothing like sending them a commercial on TV.
The next tip is a technical one related to how you have to craft your HTML. Lamson’s
lamson.htmlAPI actually
helps you get this right, but you need to understand it in order to appreciate what
it does for you.
The rule when crafting your HTML is that you need to code for browsers of circa 1995
with no ability to use style tags or script tags. Most of the various HTML viewing
clients strip or disable these making them useless. This means you have to put all
CSS stylings inline into your HTML, and you have to rely on “taboo” tags like center
and table for your layout.
The Structure Of HtmlMail
The lamson.html.HtmlMail class
is responsible for generating all your HTML emails. How it works is you give construct one
with an “outer template” and a CleverCSS template as your
CSS stylesheet. This builds a generator that you then hand an internal version of your email
to generate the content for each user.
This combination of outer template+CleverCSS and inner markdown content means that you can
use the markdown content also as your text version, since markdown actually looks half-decent
as a text format.
Here’s a simple example that shows this process in action:
generator = html.HtmlMail("style.css", "html_test.html", {})
resp = generator.respond({}, "content.markdown",
From="zedshaw@zedshaw.com",
To="zedshaw@zedshaw.com",
Subject="Test of an HTML mail.",
Body="content.markdown")
First, we make a generator passing in the CleverCSS stylesheet style.css,
and the Jinja2 HTML template html_test.html. Then we call HtmlMail.respond
to craft a lamson.mail.MailResponse
that you can then hand to lamson.server.Relay
for delivery.
There’s also a couple of tricks in the above email. First, notice that the
second parameter is “content.markdown”, but that we also pass in the Body=“content.markdown”.
This little convenience tells the HtmlMail object that you want to reuse the
raw content.markdown file as the text/plain version of the email.
Finally, once you make the generator you can keep calling respond to spit out
each message you want.
The HTMLCSS Conversion
None of the above code shows you what is the nicest part of this API. The lamson.html
API will actually take plain HTML and your CleverCSS and insert the style attributes
into it for you. Let’s look at an example from the unit test (that’s disgusting).
First we have the CleverCSS template:
body:
margin: 10
padding: 20
background: green - 30
color: blue
h1:
font-size: 3em
h2:
font-size: 2em
color: yellow
h3:
font-size: 1em
p:
padding: 0.3em
background: red
h2:
color: yellow
#bright:
background: black
color: white
.dull:
background: gray
color: black
Then we have the raw original HTML:
<html>
<head>
<title>{{ title }}</title>
</head>
<body style="background: magenta">
<h1 class="bright">{{ title }}</h1>
{{ content }}
<h3 id="dull">All done.</h3>
</body>
</html>
Notice this is a template too, and that {{ content }} is your content.markdown file
from the earlier discussion.
Now when you run this (including the content.markdown not shown here), Lamson produces
this:
<html>
<head>
<title></title>
</head>
<body style="background: magenta; margin: 10; padding: 20; background: #006200; color: blue">
<h1 class="bright" style="font-size: 3em; background: black; color: white"></h1>
<h1 style="font-size: 3em">Hello</h1>
<p style="padding: 0.3em; background: red">I would <em>love</em> for you to tell me what is going on here joe. NOW!</p>
<h2 style="font-size: 2em; color: yellow; color: yellow">Alright</h2>
<p style="padding: 0.3em; background: red">This is the best I can come up with.</p>
<p style="padding: 0.3em; background: red">Zed</p>
<h3 id="dull" style="font-size: 1em; background: gray; color: black">All done.</h3>
</body>
</html>
Which, if you code for a Web 2.0 company is probably making your eyes bleed Dijon mustard, but it
works. Lamson has walked your HTML and inserted all the style tags it could, including
keeping any you already had there.
Conclusion
With Lamson HTML email API you should be able to blast out all the wonderful HTML you need
to prop up your sales needs for years to come. It does the best it can to make it easy
to still work in a modern web methodology, but produce the nasty HTML that has the highest
chance of working in most email clients.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/html_email_generation.txt 0000644 0000765 0000024 00000014137 11243706470 026650 0 ustar zedshaw staff Title: HTML Email Generation
HTML Email is apparently the killer feature for everyone who uses email.
The first, second, and third question I get asked when I tell people
about Lamson is, "Does it do HTML email?!" Yes it does, and hopefully
in a very nice clean way that'll make blasting out all that great
marketing material your users *love* easier than ever.
bq. The fourth question is whether Lamson handles bounces. "Yes it does.":/docs/bounce_detection.html
h2. A Few Tips About HTML Email
First, you *must* educate your marketing people that your user's "Inbox is not a TV.":http://myinboxisnota.tv/
I actually run "myinboxisnota.tv":http://myinboxisnota.tv/ just to make a point that people who
do marketing have the wrong idea of people's Inbox experience. It's your job as the hacker to
educate them and make sure that they know users actually mostly hate HTML email, and that sending
them an HTML email is nothing like sending them a commercial on TV.
The next tip is a technical one related to how you have to craft your HTML. Lamson's
"lamson.html":http://lamsonproject.org/docs/api/lamson.html-module.html API actually
helps you get this right, but you need to understand it in order to appreciate what
it does for you.
The rule when crafting your HTML is that you need to code for browsers of circa 1995
with no ability to use @style@ tags or @script@ tags. Most of the various HTML viewing
clients strip or disable these making them useless. This means you have to put all
CSS stylings inline into your HTML, and you have to rely on "taboo" tags like @center@
and @table@ for your layout.
h2. The Structure Of HtmlMail
The "lamson.html.HtmlMail":http://lamsonproject.org/docs/api/lamson.html.HtmlMail-class.html class
is responsible for generating all your HTML emails. How it works is you give construct one
with an "outer template" and a "CleverCSS":http://sandbox.pocoo.org/clevercss/ template as your
CSS stylesheet. This builds a generator that you then hand an internal version of your email
to generate the content for each user.
This combination of outer template+CleverCSS and inner markdown content means that you can
use the markdown content also as your text version, since markdown actually looks half-decent
as a text format.
Here's a simple example that shows this process in action:
generator = html.HtmlMail("style.css", "html_test.html", {})
resp = generator.respond({}, "content.markdown",
From="zedshaw@zedshaw.com",
To="zedshaw@zedshaw.com",
Subject="Test of an HTML mail.",
Body="content.markdown")
First, we make a generator passing in the CleverCSS stylesheet @style.css@,
and the Jinja2 HTML template @html_test.html@. Then we call @HtmlMail.respond@
to craft a "lamson.mail.MailResponse":http://lamsonproject.org/docs/api/lamson.mail.MailResponse-class.html
that you can then hand to "lamson.server.Relay":http://lamsonproject.org/docs/api/lamson.server.Relay-class.html
for delivery.
There's also a couple of tricks in the above email. First, notice that the
second parameter is "content.markdown", but that we also pass in the Body="content.markdown".
This little convenience tells the HtmlMail object that you want to reuse the
raw @content.markdown@ file as the text/plain version of the email.
Finally, once you make the generator you can keep calling @respond@ to spit out
each message you want.
h2. The HTML CSS Conversion
None of the above code shows you what is the nicest part of this API. The lamson.html
API will actually take plain HTML and your CleverCSS and insert the @style@ attributes
into it for you. Let's look at an example from the unit test (that's disgusting).
First we have the CleverCSS template:
body:
margin: 10
padding: 20
background: green - 30
color: blue
h1:
font-size: 3em
h2:
font-size: 2em
color: yellow
h3:
font-size: 1em
p:
padding: 0.3em
background: red
h2:
color: yellow
#bright:
background: black
color: white
.dull:
background: gray
color: black
Then we have the raw original HTML:
<html>
<head>
<title>{{ title }}</title>
</head>
<body style="background: magenta">
<h1 class="bright">{{ title }}</h1>
{{ content }}
<h3 id="dull">All done.</h3>
</body>
</html>
Notice this is a template too, and that {{ content }} is your @content.markdown@ file
from the earlier discussion.
Now when you run this (including the content.markdown not shown here), Lamson produces
this:
<html>
<head>
<title></title>
</head>
<body style="background: magenta; margin: 10; padding: 20; background: #006200; color: blue">
<h1 class="bright" style="font-size: 3em; background: black; color: white"></h1>
<h1 style="font-size: 3em">Hello</h1>
<p style="padding: 0.3em; background: red">I would <em>love</em> for you to tell me what is going on here joe. NOW!</p>
<h2 style="font-size: 2em; color: yellow; color: yellow">Alright</h2>
<p style="padding: 0.3em; background: red">This is the best I can come up with.</p>
<p style="padding: 0.3em; background: red">Zed</p>
<h3 id="dull" style="font-size: 1em; background: gray; color: black">All done.</h3>
</body>
</html>
Which, if you code for a Web 2.0 company is probably making your eyes bleed Dijon mustard, but it
works. Lamson has walked your HTML and inserted all the style tags it could, including
keeping any you already had there.
h2. Conclusion
With Lamson HTML email API you should be able to blast out all the wonderful HTML you need
to prop up your sales needs for years to come. It does the best it can to make it easy
to still work in a modern web methodology, but produce the nasty HTML that has the highest
chance of working in most email clients.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/index.html 0000644 0000765 0000024 00000020375 11313464306 023554 0 ustar zedshaw staff
LamsonProject: Lamson Project Documentation
This is the Lamson documentation, organized into categories from “newbie” to “expert”. You should
also check out the screen casts available which might help you if you’re the more mediated
type.
The documentation is only updated after major releases, and it may not have the best information.
If you are following the docs and they’re wrong, then send an email to lamson@librelist.com
and report it.
Finally, every document is also uploaded with it’s text version.
Deploying Lamson Level 1 — This is for when you’re getting more serious about deployment. Involves building a completely separate virtualenv+python for Lamson and shows deploying oneshotblog in it.
Deploying Lamson Level 2 — At this level of deployment you are running multiple sites on the same server using a virtualhost setup, and you have each application installed under its own user.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/index.txt 0000644 0000765 0000024 00000006277 11243570044 023433 0 ustar zedshaw staff Title: Lamson Project Documentation
This is the Lamson documentation, organized into categories from "newbie" to "expert". You should
also check out the "screen casts":/videos/ available which might help you if you're the more mediated
type.
The documentation is only updated after major releases, and it may not have the best information.
If you are following the docs and they're wrong, then "send an email to lamson@librelist.com":mailto:lamson@librelist.com
and report it.
Finally, every document is also uploaded with it's text version.
h2. Initial Concepts
* "Frequently Asked Question":/docs/faq.html -- Questions people have asked about Lamson.
* "Getting Started":/docs/getting_started.html -- A fast tour of getting Lamson going and doing something with it.
* "Lamson Commands":/docs/lamson_commands.html -- All of the commands Lamson supports. You can get at this with @lamson -help@.
* "Introduction To Finite State Machines":/docs/introduction_to_finite_state_machines.html -- Important to understand the simplified version of Finite State Machines Lamson uses.
h2. Advanced Concepts
* "Deferred Processing To Queues":/docs/deferred_processing_to_queues.html -- Very handy way of processing mail.
* "Writing A Custom State Store":/docs/writing_a_state_storage.html -- You'll need this if you want to store state in the database.
* "Primary Vs. Secondary Registration":/docs/primary_vs_secondary_registration.html -- The concept of doing registration in Lamson, where contacting the service the first time is the registration.
* "Hooking Into Django":/docs/hooking_into_django.html -- Shows you how to get access to a Django ORM model.
h2. Deployment
* "Deploying Lamson Level 0 With Virtualenv And Pip":/docs/lamson_virtual_env.html -- Quick instructions for setting up your Lamson in a virtualenv for your first simple deployment.
* "Deploying Lamson Level 1":/docs/deploying_lamson.html -- This is for when you're getting more serious about deployment. Involves building a completely separate virtualenv+python for Lamson and shows deploying oneshotblog in it.
* "Deploying Lamson Level 2":/docs/deploying_lamson_level_2.html -- At this level of deployment you are running multiple sites on the same server using a virtualhost setup, and you have each application installed under its own user.
h2. Deployment: Examples
* "Deploying OneShotBlog":/docs/deploying_oneshotblog.html
h2. Specific Features
* "Unit Testing":/docs/unit_testing.html -- Lamson has a few simple things to help write better mail specific unit tests.
* "Confirmations":/docs/confirmations.html -- Confirming a user so that you validate they are an actual email address.
* "Filtering Spam":/docs/filtering_spam.html -- How to use Lamson's spam blocking features. It's easy to use, but a bit hairy to setup.
* "Bounce Detection":/docs/bounce_detection.html -- Using Lamson's bounce message parser to handle bounces.
* "Unicode Encoding And Decoding":/docs/unicode_encoding_and_decoding.html -- How Lamson decodes the nastiest email into Unicode, and then converts Unicode back into a clean email for sending.
* "HTML Email Generation":/docs/html_email_generation.html -- Using Lamson's HTML email generation library to send out HTML to annoy everyone with.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/introduction_to_finite_state_machines.html 0000644 0000765 0000024 00000042264 11242713720 032274 0 ustar zedshaw staff
LamsonProject: A Painless Introduction To Finite State Machines
Lamson uses the concept of a Finite State Machine to do the internal processing
and keep track of what it should do next. You don’t necessarily need to know
the details of how a FSM works, but it helps you if you want to know enough
about Lamson to do advanced work. Most people could be blissfully unaware of
state machines and still do plenty of work with Lamson.
Your Computer Science Class Sucked
When I say “finite state machine” everyone reads it as:
Yes, the way your professor explained FSM makes them much harder than they are
in practice. You were probably thrown fairly random sounding terms like
“edge”, “node”, “transitions”, “acceptors”, “recognizers”, and “transducers”.
You were probably shown graphs with lines and circles and told confusingly that
the lines were edges and the circles were nodes, but that the edges were states
unless you used this kind where the edges were transitions and that kind where
the circles were states.
Alright, just forget about that because they aren’t really that complicated. A
practical finite state machine is basically four things:
A bunch of functions, or things that need to get done.
A bunch of events, or reasons to call these functions.
Some piece of data that tracks the “state” this bunch of functions is in.
Code inside the functions that says how to “transition” or “change” into the next state for further processing.
That is really all there is to every FSM. Sure you can change around what a
state is, add code that runs on a transition, or make FSM that “inherit” from
other FSM, but in the end they are all:
functions, events, states, transitions
That is all there is to it, no matter how you slice these or dice these up.
Implementing A Practical FSM
Now that we’ve brought the fear level down, let’s actually make one. This will
be a little toy FSM, but it will get you started and it won’t be too far off
from what Lamson uses.
We are going to implement your classic email confirmation system. This system
usually works like this:
You receive an email at some service, let’s say users-subscribe@test.com.
You reply to the sender with a confirmation message.
They reply with an email to your confirmation’s random address.
You receive this, approve the handshake, and make them subscribed with a welcome email.
They send an email, and since they are subscribed, you post it to the list.
This is a typical operation on most mailing lists, but with a state machine it
turns out to be fairly trivial to implement.
First, we need to look at the above conversation and think about what is more
clear: states the user will be in, or events that they will generate. In this
case I’d go with the states they will be in during the conversation:
START — Every machine has one of these.
PENDING — We sent the confirm, and waiting for their reply.
POSTING — They replied to the confirm and can now post.
Notice I gave each state a verb. You are saying what state this user is in.
“They are currently posting.” You don’t say what they “did”. Now, sometimes
it just makes sense to do it differently, so don’t be a slave to this setup.
I’ve just found it helps keep things consistent if you name the states after
what they are currently doing.
Now that we know what kind of states we’re dealing with we need to solve the
next two parts of the puzzle: events or transitions. In an email application
I like to start with the events, as these are the email addresses and messages
they will be sending me. What we do now is for each state, list the addresses
we’ll consider an “event” for that state.
START — (list_name)-subscribe@test.com
PENDING — (list_name)-confirm-(id)@test.com
POSTING — (list_name)@test.com
This is all the “events” we need right now, written out as email addresses.
If we were to add the ability to unsubscribe, then we would add an event
for (list_name)-unsubscribe@test.com to POSTING so that we could transition
them from POSTING to say, SLEEPING.
We now have our states, and the events that each state answers. Last step is
to figure out what state “transitions” to which other state.
Write The Functions
At this point, our FSM is simple enough that we could just write the functions,
and the logic of each function would dictate the transitions. However, if you
had a larger number of states and events you would want to sit down and draw a
diagram or a make a table of the transitions before you wrote some code.
To start our functions we’ll just name them after the states and put the events
they handle at the top as pseudo code event decorators:
@event("(list_name)-subscribe@test.com")
def START(...):
"""Initial setup of the user."""
...
@event("(list_name)-confirm-(id)@test.com")
def PENDING(...):
"""Waiting for them to confirm."""
...
@event("(list_name)@test.com")
def POSTING(...):
"""They are posting, anything they send we post."""
...
This is abbreviating the syntax quite a bit, is not functioning Python code but
it is pretty close. Instead of event you would have a route decorator and
it would have a few regexes. Otherwise it’s about the same.
We could also say that each state handles multiple events, which is what
you would do if POSTING handled the “unsubscribe” requests.
Add Logic And Transitions
The final piece, and the part you’ll spend the longest getting right, is
filling in the logic and making the transitions happen. How would you indicate
where each state should go next? Remember that each state is a simple Python
function, and that to “transition” means to change to another state. Well, we
have to tell whatever is running the state machine to run a different
function next time.
Easiest way to do that is for our handlers to just return the next function.
The “runner” will then take that, store it somewhere, and the next time an
event comes the runner will load the next function to run.
Here’s the psuedo code to do just that:
@event("(list_name)-subscribe@test.com")
def START(...):
"""Initial setup of the user."""
send_confirmation()
return PENDING
@event("(list_name)-confirm-(id)@test.com")
def PENDING(...):
"""Waiting for them to confirm."""
if check_confirmation_is_good():
send_welcome()
return POSTING
else:
ignore_them()
@event("(list_name)@test.com")
def POSTING(...):
"""They are posting, anything they send we post."""
deliver_message_to_all()
return POSTING
Right away you can see that we change to the next state by just returning the
actual function to call next. When the next event comes in, the runner matches
it to the right function, calls it, and then sets up for the next one.
If you look at PENDING, you can see that it either returns POSTING if they
confirm correctly or it just ignores them. You could also transition to
“return ERROR” if you wanted to put them in an error state and send a different
message.
Looking at POSTING, you see that it just keeps returning itself to indicate
that it is staying there. If you had POSTING process “unsubscribes” then it
would simply do the unsubscribe confirm, and then transition to UNSUBSCRIBING.
That state function would then check the confirmation and unsubscribe them,
transitioning to something like DEAD or SLEEPING.
An “optimization” in Lamson is that if a state function doesn’t return
anything then it’s assumed to just want to stay in that state. In this case
POSTING could have no return and it would work the same.
Jump To vs. Transition
You now know pretty much everything you need to handle FSM except for a tiny
corner case. It is typically called the “epsilon transition” which basically
means “transition to that state without an explicit event”. When you use this
is when your event needs to fire off the code for the next transition right
now, rather than waiting for the next event from the user.
In the above pseudo code this is simply done by actually calling that
transition function and returning whatever it returns. Let’s say we wanted to
have PENDING also call POSTING with the original message they sent:
Notice that we changed the return POSTING to a return POSTING(...) which
returns the results of calling POSTING.
That’s it, you now know epsilon transitions. Man, that was tough.
The danger of this is if you don’t have your FSM carefully mapped out,
then you could call a state that loops back to your state and you’re in an
endless loop. Watch for that.
Why Finite State Machines Anyway
Finite State Machines like this are very powerful because they behave in ways
that are consistent, easily debugged, and intelligent. Because the decision of
how each step in the chain of events is controlled by a constrained set of
states, events, and transitions, you can actually avoid many bugs you’d get in
regular classes and objects.
For example, if you were to receive another subscribe message while there is a
confirmation pending, then that event (email address) isn’t recognize and just
ignored. You would see it in the logs, and see that your FSM stayed in the
PENDING state. If you then wanted PENDING to handle additional requests it’s a
simple matter of adding that event and writing the code.
The way to think of the a FSM is it is like an object that has a white list of
functions, parameters, AND allowed values for its private data. If an FSM
tries to change to a state that doesn’t exist it’s an error. If it gets an
event it doesn’t know it ignores it. If you try to transition wrong you see it
in the logs, or it’s an error.
FSM also have the debugging advantage of showing you the history of not only
what states were called, but why they were called and what they did next.
You will see the entire conversation and can pinpoint exactly where it went
wrong.
However, the most important reason to use FSM is because this is how email and
asynchronous conversations work. When you have a conversation with someone
there is state involved. You don’t have to start the entire conversation from
scratch at the start of each sentence. Instead you remember what the person
said and what state you are in (angry, sad, happy) and that controls what you
do next (punch them, run away, hug them) based on the events they send (“screw
you”, “you’re dead”, “i love you”).
The use of an FSM will make your Lamson applications seem like magic. They
will behave more like smart systems that just seems to know like a wizard what
should happen next, and why it should happen that way.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/introduction_to_finite_state_machines.txt 0000644 0000765 0000024 00000025451 11242713711 032146 0 ustar zedshaw staff Title: A Painless Introduction To Finite State Machines
Lamson uses the concept of a Finite State Machine to do the internal processing
and keep track of what it should do next. You don't necessarily need to know
the details of how a FSM works, but it helps you if you want to know enough
about Lamson to do advanced work. Most people could be blissfully unaware of
state machines and still do plenty of work with Lamson.
h2. Your Computer Science Class Sucked
When I say "finite state machine" everyone reads it as:
bq. "FINITE STATE MACHINE!!!!! OMGWTFBBQ THOSE ARE HARD!!!!"
Yes, the way your professor explained FSM makes them much harder than they are
in practice. You were probably thrown fairly random sounding terms like
"edge", "node", "transitions", "acceptors", "recognizers", and "transducers".
You were probably shown graphs with lines and circles and told confusingly that
the lines were edges and the circles were nodes, but that the edges were states
unless you used this kind where the edges were transitions and that kind where
the circles were states.
Alright, just forget about that because they aren't really that complicated. A
practical finite state machine is basically four things:
* A bunch of functions, or things that need to get done.
* A bunch of events, or reasons to call these functions.
* Some piece of data that tracks the "state" this bunch of functions is in.
* Code inside the functions that says how to "transition" or "change" into the next state for further processing.
That is really all there is to every FSM. Sure you can change around what a
state is, add code that runs on a transition, or make FSM that "inherit" from
other FSM, but in the end they are all:
bq. functions, events, states, transitions
That is all there is to it, no matter how you slice these or dice these up.
h2. Implementing A Practical FSM
Now that we've brought the fear level down, let's actually make one. This will
be a little toy FSM, but it will get you started and it won't be too far off
from what Lamson uses.
We are going to implement your classic email confirmation system. This system
usually works like this:
# You receive an email at some service, let's say users-subscribe@test.com.
# You reply to the sender with a confirmation message.
# They reply with an email to your confirmation's random address.
# You receive this, approve the handshake, and make them subscribed with a welcome email.
# They send an email, and since they are subscribed, you post it to the list.
This is a typical operation on most mailing lists, but with a state machine it
turns out to be fairly trivial to implement.
First, we need to look at the above conversation and think about what is more
clear: states the user will be in, or events that they will generate. In this
case I'd go with the states they will be in during the conversation:
* START -- Every machine has one of these.
* PENDING -- We sent the confirm, and waiting for their reply.
* POSTING -- They replied to the confirm and can now post.
Notice I gave each state a verb. You are saying what state this user is in.
"They are currently posting." You don't say what they "did". Now, sometimes
it just makes sense to do it differently, so don't be a slave to this setup.
I've just found it helps keep things consistent if you name the states after
what they are currently doing.
Now that we know what kind of states we're dealing with we need to solve the
next two parts of the puzzle: events or transitions. In an email application
I like to start with the events, as these are the email addresses and messages
they will be sending me. What we do now is for each state, list the addresses
we'll consider an "event" for that state.
* START -- (list_name)-subscribe@test.com
* PENDING -- (list_name)-confirm-(id)@test.com
* POSTING -- (list_name)@test.com
This is all the "events" we need right now, written out as email addresses.
bq. If we were to add the ability to unsubscribe, then we would add an event
for (list_name)-unsubscribe@test.com to POSTING so that we could transition
them from POSTING to say, SLEEPING.
We now have our states, and the events that each state answers. Last step is
to figure out what state "transitions" to which other state.
h2. Write The Functions
At this point, our FSM is simple enough that we could just write the functions,
and the logic of each function would dictate the transitions. However, if you
had a larger number of states and events you would want to sit down and draw a
diagram or a make a table of the transitions before you wrote some code.
To start our functions we'll just name them after the states and put the events
they handle at the top as pseudo code @event@ decorators:
@event("(list_name)-subscribe@test.com")
def START(...):
"""Initial setup of the user."""
...
@event("(list_name)-confirm-(id)@test.com")
def PENDING(...):
"""Waiting for them to confirm."""
...
@event("(list_name)@test.com")
def POSTING(...):
"""They are posting, anything they send we post."""
...
This is abbreviating the syntax quite a bit, is not functioning Python code but
it is pretty close. Instead of @event@ you would have a @route@ decorator and
it would have a few regexes. Otherwise it's about the same.
bq. We could also say that each state handles multiple events, which is what
you would do if POSTING handled the "unsubscribe" requests.
h2. Add Logic And Transitions
The final piece, and the part you'll spend the longest getting right, is
filling in the logic and making the transitions happen. How would you indicate
*where* each state should go next? Remember that each state is a simple Python
function, and that to "transition" means to change to another state. Well, we
have to tell whatever is running the state machine to run a *different*
function next time.
Easiest way to do that is for our handlers to just return the next function.
The "runner" will then take that, store it somewhere, and the next time an
event comes the runner will load the next function to run.
Here's the psuedo code to do just that:
@event("(list_name)-subscribe@test.com")
def START(...):
"""Initial setup of the user."""
send_confirmation()
return PENDING
@event("(list_name)-confirm-(id)@test.com")
def PENDING(...):
"""Waiting for them to confirm."""
if check_confirmation_is_good():
send_welcome()
return POSTING
else:
ignore_them()
@event("(list_name)@test.com")
def POSTING(...):
"""They are posting, anything they send we post."""
deliver_message_to_all()
return POSTING
Right away you can see that we change to the next state by just returning the
actual function to call next. When the next event comes in, the runner matches
it to the right function, calls it, and then sets up for the next one.
If you look at PENDING, you can see that it either returns POSTING if they
confirm correctly or it just ignores them. You could also transition to
"return ERROR" if you wanted to put them in an error state and send a different
message.
Looking at POSTING, you see that it just keeps returning itself to indicate
that it is staying there. If you had POSTING process "unsubscribes" then it
would simply do the unsubscribe confirm, and then transition to UNSUBSCRIBING.
That state function would then check the confirmation and unsubscribe them,
transitioning to something like DEAD or SLEEPING.
bq. An "optimization" in Lamson is that if a state function doesn't return
anything then it's assumed to just want to stay in that state. In this case
POSTING could have no return and it would work the same.
h2. Jump To vs. Transition
You now know pretty much everything you need to handle FSM except for a tiny
corner case. It is typically called the "epsilon transition" which basically
means "transition to that state without an explicit event". When you use this
is when your event needs to fire off the code for the *next* transition right
now, rather than waiting for the next event from the user.
In the above pseudo code this is simply done by actually calling that
transition function and returning whatever it returns. Let's say we wanted to
have PENDING also call POSTING with the original message they sent:
Notice that we changed the @return POSTING@ to a @return POSTING(...)@ which
returns the results of calling POSTING.
That's it, you now know epsilon transitions. Man, that was tough.
bq. The danger of this is if you don't have your FSM carefully mapped out,
then you could call a state that loops back to your state and you're in an
endless loop. Watch for that.
h2. Why Finite State Machines Anyway
Finite State Machines like this are very powerful because they behave in ways
that are consistent, easily debugged, and intelligent. Because the decision of
how each step in the chain of events is controlled by a constrained set of
states, events, and transitions, you can actually avoid many bugs you'd get in
regular classes and objects.
For example, if you were to receive another subscribe message while there is a
confirmation pending, then that event (email address) isn't recognize and just
ignored. You would see it in the logs, and see that your FSM stayed in the
PENDING state. If you then wanted PENDING to handle additional requests it's a
simple matter of adding that event and writing the code.
The way to think of the a FSM is it is like an object that has a white list of
functions, parameters, AND allowed values for its private data. If an FSM
tries to change to a state that doesn't exist it's an error. If it gets an
event it doesn't know it ignores it. If you try to transition wrong you see it
in the logs, or it's an error.
FSM also have the debugging advantage of showing you the history of not only
what states were called, but *why* they were called and what they did next.
You will see the entire conversation and can pinpoint exactly where it went
wrong.
However, the most important reason to use FSM is because this is how email and
asynchronous conversations work. When you have a conversation with someone
there is state involved. You don't have to start the entire conversation from
scratch at the start of each sentence. Instead you remember what the person
said and what state you are in (angry, sad, happy) and that controls what you
do next (punch them, run away, hug them) based on the events they send ("screw
you", "you're dead", "i love you").
The use of an FSM will make your Lamson applications seem like magic. They
will behave more like smart systems that just seems to know like a wizard what
should happen next, and *why* it should happen that way.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/lamson_commands.html 0000644 0000765 0000024 00000021637 11242706607 025625 0 ustar zedshaw staff
LamsonProject: Available Lamson Commands
The following is also available by running lamson help and you can get the
help for each individual command with lamson help -for COMMAND replacing
COMMAND with one of these listed below.
The format for the printed options show default options as an actual setting,
and required options as a CAPITALIZED setting you must give. For example, in the
send command:
The options -port, -host, -debug, and -file have default settings, but -sender,
-to, -subject and -body require a STRing or EMAIL. Notice also that -file
defaults to False which you can change by just including -file (that
toggles it true).
lamson blast
Given a maildir, this command will go through each email
and blast it at your server. It does nothing to the message, so
it will be real messages hitting your server, not cleansed ones.
lamson cleanse
Uses Lamson mail cleansing and canonicalization system to take an
input maildir (or mbox) and replicate the email over into another
maildir. It's used mostly for testing and cleaning.
lamson gen
Generates various useful things for you to get you started.
lamson gen -project STR -FORCE False
lamson help
Prints out help for the commands.
lamson help
You can get help for one command with:
lamson help -for STR
lamson log
Runs a logging only server on the given hosts and port. It logs
each message it receives and also stores it to the run/queue
so that you can make sure it was received in testing.
lamson log -port 8825 -host 127.0.0.1 \
-pid ./run/log.pid -chroot False \
-chdir "." -umask False -uid False -gid False \
-FORCE False
If you specify a uid/gid then this means you want to first change to
root, set everything up, and then drop to that UID/GID combination.
This is typically so you can bind to port 25 and then become "safe"
to continue operating as a non-root user.
If you give one or the other, this it will just change to that
uid or gid without doing the priv drop operation.
lamson queue
Let's you do most of the operations available to a queue.
lamson queue (-pop | -get | -remove | -count | -clear | -keys) -name run/queue
lamson restart
Simply attempts a stop and then a start command. All options for both
apply to restart. See stop and start for options available.
lamson routes
Prints out valuable information about an application's routing configuration
after everything is loaded and ready to go. Helps debug problems with
messages not getting to your handlers. Path has the search paths you want
separated by a ':' character, and it's added to the sys.path.
lamson routes -path $PWD -- config.testing -test ""
It defaults to running your config.testing to load the routes.
If you want it to run the config.boot then give that instead:
lamson routes -- config.boot
You can also test a potential target by doing -test EMAIL.
lamson send
Sends an email to someone as a test message.
See the sendmail command for a sendmail replacement.
lamson send -port 8825 -host 127.0.0.1 -debug 1 \
-sender EMAIL -to EMAIL -subject STR -body STR -attach False'
lamson sendmail
Used as a testing sendmail replacement for use in programs
like mutt as an MTA. It reads the email to send on the stdin
and then delivers it based on the port and host settings.
lamson sendmail -port 8825 -host 127.0.0.1 -debug 0 -- [recipients]
lamson start
Runs a lamson server out of the current directory:
lamson start -pid ./run/smtp.pid -FORCE False -chroot False -chdir "." \
-umask False -uid False -gid False -boot config.boot
lamson status
Prints out status information about lamson useful for finding out if it's
running and where.
lamson status -pid ./run/smtp.pid
lamson stop
Stops a running lamson server. Give -KILL True to have it
stopped violently. The PID file is removed after the
signal is sent. Give -ALL the name of a run directory and
it will stop all pid files it finds there.
lamson stop -pid ./run/smtp.pid -KILL False -ALL False
lamson version
Prints the version of Lamson, the reporitory revision, and the
file it came from.
lamson web
Starts a very simple files only web server for easy testing of applications
that need to make some HTML files as the result of their operation.
If you need more than this then use a real web server.
lamson web -basedir "." -port 8888 -host '127.0.0.1'
This command doesn't exit so you can view the logs it prints out.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
The following is also available by running lamson help and you can get the
help for each individual command with lamson help -for COMMAND replacing
COMMAND with one of these listed below.
The format for the printed options show default options as an actual setting,
and required options as a CAPITALIZED setting you must give. For example, in the
send command:
The options -port, -host, -debug, and -file have default settings, but -sender,
-to, -subject and -body require a STRing or EMAIL. Notice also that -file
defaults to False which you can change by just including -file (that
toggles it true).
lamson blast
Given a maildir, this command will go through each email
and blast it at your server. It does nothing to the message, so
it will be real messages hitting your server, not cleansed ones.
lamson cleanse
Uses Lamson mail cleansing and canonicalization system to take an
input maildir (or mbox) and replicate the email over into another
maildir. It's used mostly for testing and cleaning.
lamson gen
Generates various useful things for you to get you started.
lamson gen -project STR -FORCE False
lamson help
Prints out help for the commands.
lamson help
You can get help for one command with:
lamson help -for STR
lamson log
Runs a logging only server on the given hosts and port. It logs
each message it receives and also stores it to the run/queue
so that you can make sure it was received in testing.
lamson log -port 8825 -host 127.0.0.1 \
-pid ./run/log.pid -chroot False \
-chdir "." -umask False -uid False -gid False \
-FORCE False
If you specify a uid/gid then this means you want to first change to
root, set everything up, and then drop to that UID/GID combination.
This is typically so you can bind to port 25 and then become "safe"
to continue operating as a non-root user.
If you give one or the other, this it will just change to that
uid or gid without doing the priv drop operation.
lamson queue
Let's you do most of the operations available to a queue.
lamson queue (-pop | -get | -remove | -count | -clear | -keys) -name run/queue
lamson restart
Simply attempts a stop and then a start command. All options for both
apply to restart. See stop and start for options available.
lamson routes
Prints out valuable information about an application's routing configuration
after everything is loaded and ready to go. Helps debug problems with
messages not getting to your handlers. Path has the search paths you want
separated by a ':' character, and it's added to the sys.path.
lamson routes -path $PWD -- config.testing -test ""
It defaults to running your config.testing to load the routes.
If you want it to run the config.boot then give that instead:
lamson routes -- config.boot
You can also test a potential target by doing -test EMAIL.
lamson send
Sends an email to someone as a test message.
See the sendmail command for a sendmail replacement.
lamson send -port 8825 -host 127.0.0.1 -debug 1 \
-sender EMAIL -to EMAIL -subject STR -body STR -attach False'
lamson sendmail
Used as a testing sendmail replacement for use in programs
like mutt as an MTA. It reads the email to send on the stdin
and then delivers it based on the port and host settings.
lamson sendmail -port 8825 -host 127.0.0.1 -debug 0 -- [recipients]
lamson start
Runs a lamson server out of the current directory:
lamson start -pid ./run/smtp.pid -FORCE False -chroot False -chdir "." \
-umask False -uid False -gid False -boot config.boot
lamson status
Prints out status information about lamson useful for finding out if it's
running and where.
lamson status -pid ./run/smtp.pid
lamson stop
Stops a running lamson server. Give -KILL True to have it
stopped violently. The PID file is removed after the
signal is sent. Give -ALL the name of a run directory and
it will stop all pid files it finds there.
lamson stop -pid ./run/smtp.pid -KILL False -ALL False
lamson version
Prints the version of Lamson, the reporitory revision, and the
file it came from.
lamson web
Starts a very simple files only web server for easy testing of applications
that need to make some HTML files as the result of their operation.
If you need more than this then use a real web server.
lamson web -basedir "." -port 8888 -host '127.0.0.1'
This command doesn't exit so you can view the logs it prints out.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/lamson_virtual_env.html 0000644 0000765 0000024 00000013271 11242706246 026354 0 ustar zedshaw staff
LamsonProject: Installing Lamson Into Virtualenv With Pip
The best way to install Lamson in a virtualenv is using the pip
tool to install it. This will automatically fetch
the code from bazaar, install it inside the virtualenv and fetch any missing
dependencies.
First, if you haven’t already, you’ll need to install pip. Make sure
you’ve activated the virtualenv:
. bin/activate
from the root of the virtualenv, and then run:
easy_install -U pip
If you have bazaar installed on your machine, then just do:
This’ll install the development version of lamson into a src/lamson
directory (src/ will be created if it doesn’t exist). It will also
install all of Lamson’s dependencies into the virtualenv. If you
want to update to the latest development version of Lamson, run:
cd src/lamson
bzr pull
This will fetch the latest bazaar tip from Launchpad.
If you don’t have bazaar installed and don’t want to install it, you
can grab the latest release of Lamson like this:
Remember though that if you do install it like this you’ll need to
manually update it by visiting Launchpad, finding the latest tarball
URL and `pip install`ing it.
Credits
These instructions were donated to Lamson by Zachary
Voase but contact me for
suggested improvements.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/lamson_virtual_env.txt 0000600 0000765 0000024 00000003155 11242705742 026217 0 ustar zedshaw staff Title: Installing Lamson Into Virtualenv With Pip
The best way to install Lamson in a virtualenv is using the "pip
tool":http://pip.openplans.org/ to install it. This will automatically fetch
the code from bazaar, install it inside the virtualenv and fetch any missing
dependencies.
First, if you haven't already, you'll need to install pip. Make sure
you've activated the virtualenv:
. bin/activate
from the root of the virtualenv, and then run:
easy_install -U pip
If you have bazaar installed on your machine, then just do:
This'll install the development version of lamson into a src/lamson
directory (src/ will be created if it doesn't exist). It will also
install all of Lamson's dependencies into the virtualenv. If you
want to update to the latest development version of Lamson, run:
cd src/lamson
bzr pull
This will fetch the latest bazaar tip from Launchpad.
If you don't have bazaar installed and don't want to install it, you
can grab the latest release of Lamson like this:
Remember though that if you do install it like this you'll need to
manually update it by visiting Launchpad, finding the latest tarball
URL and `pip install`ing it.
h2. Credits
These instructions were donated to Lamson by "Zachary
Voase":http://disturbyte.github.com but "contact me":/contact.html for
suggested improvements.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/primary_vs_secondary_registration.html 0000644 0000765 0000024 00000027004 11242714125 031473 0 ustar zedshaw staff
LamsonProject: Primary vs. Secondary Registration
When you design a Lamson application the question of “registration” comes up
almost immediately. The question is one of how you find out who a person is so
that you can interact with them. In the web world this involves a form of some
kind asking for a login and password at a minimum. Some sites even go so far
as to require a sign-up process before they let you see anything about the
site, or even their marketing material.
On the web this kind of registration is what I like to call “Primary
Registration”. It comes from the concept of primary vs. secondary data, or
primary vs. secondary analysis. In the world of measurement a primary source
is one that you gathered the information from directly, usually by giving them
a questionaire. A secondary source is one that you gathered in some indirect
way, either by evaluating past research in a new way, data mining, or simply
unobtrusive data collection.
With a web site you must have a primary registration system. There is no
implied identity in a browser that is consistent for a user, so you need to ask
them for some information and associate their stateless browser to a cookie or
similar tracking mechanism. You have to do this for each session you start
with them, and to register them if your service requires some kind of password
protection.
In the world of the web primary registration is just how things are done.
Secondary Registration
Lamson give you the ability to do Secondary Registration where the act of
interacting with your service is the registration. A typical primary
registrationg process will ask for a user’s email address to identify the user,
but in Lamson the user actively gives your service their email address to start
playing. Their first message to your service actually has all the information
you need to sign them up and give them their first taste of your service.
In fact, a user’s identity is so baked into the email protocols that your
Lamson server has almost the inverse problem: it’s too easy to fake an
identity, so you need to confirm their subscription and important actions.
This confirmation is traditionally nothing more than a reply asking them to
reply with some random number in the address. If you get a reply to this
auto-generated email address then you assume they are real.
This “random number confirmation handshake” is important because SMTP
allows anyone with an internet connection to lie about who they are and craft a
fake email claiming to be someone else. The confirmation assures you that the
supposed sender has at least received an email in their inbox and that it had
the random number you generated in the reply address. A determined hacker
could also get past that, but if you have that situation you are dealing with
more problems than just confirmations.
Once you do though you are fairly assured that they are identified and using
your service. It ends up being the most low effort entry to your service you
could possibly devise without making the service unsafe.
Comparing Two Designs
When you sit down to build a Lamson application you may still think about
registration in the primary way, but a secondary registration is typically more
natural for an email service. To illustrate the point, let’s say you are
creating an online game using Lamson, and you want people to register for the
game.
With a primary registration you would approach the problem by making your game
open only to registered users. If they send an email to your game you send a
single reply pointing them at the web page they can use to sign up. You also
then take every email and pass it through a filter that checks that the user is
registered in your subscription database. This is mostly how you would do it
in a web application.
Maybe there is a legitimate reason to force a user to a web page to give a
password and other information before using the service. If that’s the case,
then go for it, but consider trying to make the entry to your service a
secondary registration that then asks them to do a full registration to get
more features.
With a secondary registration system, you are using Lamson’s ability to track
the state of every user to know whether they are registered or not. When your
game receives their first email, it would transition to an UNKNOWN state and
reply with a confirmation email. Once they reply to the confirm email you
transition from UNKNOWN to the rest of the game since they are now
registered. Every interaction after that will simply know they are registered
because they are in the right state.
This has a big impact on your game’s design because now you do not need to
check the database on every received email to confirm their identity. Their
email address and their state in your state’s storage tells you that
implicitly.
You can also extend this to other requirements for your system. You can use
their known state to transition them from free to paid quality services, to
suspended states, to pause them after a bounce, and back to normal activity.
Invite Only Systems
An invite only system can also use the secondary registration technique by
having the inviter’s invitation request create a record for the invitee that’s
in the right start state. When the inviter tells you to invite their friend,
you create a state for the friend that says INVITED and use the invite as a
sort of implied confirmation email. When the invitee replies to this invite
email, you transition them to the rest of the system as they’ve now registered.
Another way to think of an invite only system in Lamson is as a 3rd-party
confirmation. Rather than the target user (the invitee) prompting a
confirmation, it is their friend sending the invite (the inviter) who triggers
the confirmation process on behalf of their buddy.
An additional bonus with Lamson is that the state of the invitee is known, so
if the inviter tries to send them again, or if someone else does, then you know
they already were sent an invite. You can just ignore the request and
avoid spamming people. You can also run reports to find out your
conversion rates by simply looking for everyone in the various states and
counting.
Security And Design Issues
Doing these kind of secondary registrations and implied identity does come with
an extra security penalty. Unless your system is incorporating S-MIME or GPG
you won’t have a solid way to identify the sender. You don’t have this on most
other internet services either so it’s not much of a loss. At least with email
you have a higher assurance that this is a real person.
When you design your service it’s a good idea to:
Either make everything the user does non-destructive,
Send confirmations for anything desctructive,
Or bounce them to a web site for more complex interactions and security.
You should also make any confirmations you send easily replied to, but have a
good random number in them that you remember for later. In fact, if you use
the “lamson.confirm” API you get all of this for free and done the right way.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/primary_vs_secondary_registration.txt 0000644 0000765 0000024 00000015756 11242714123 031357 0 ustar zedshaw staff Title: Primary vs. Secondary Registration
When you design a Lamson application the question of "registration" comes up
almost immediately. The question is one of how you find out who a person is so
that you can interact with them. In the web world this involves a form of some
kind asking for a login and password at a minimum. Some sites even go so far
as to require a sign-up process *before* they let you see anything about the
site, or even their marketing material.
On the web this kind of registration is what I like to call "Primary
Registration". It comes from the concept of primary vs. secondary data, or
primary vs. secondary analysis. In the world of measurement a primary source
is one that you gathered the information from directly, usually by giving them
a questionaire. A secondary source is one that you gathered in some indirect
way, either by evaluating past research in a new way, data mining, or simply
unobtrusive data collection.
With a web site you *must* have a primary registration system. There is no
implied identity in a browser that is consistent for a user, so you need to ask
them for some information and associate their stateless browser to a cookie or
similar tracking mechanism. You have to do this for each session you start
with them, and to register them if your service requires some kind of password
protection.
In the world of the web primary registration is just how things are done.
h2. Secondary Registration
Lamson give you the ability to do *Secondary Registration* where the act of
interacting with your service *is* the registration. A typical primary
registrationg process will ask for a user's email address to identify the user,
but in Lamson the user actively gives your service their email address to start
playing. Their first message to your service actually has all the information
you need to sign them up and give them their first taste of your service.
In fact, a user's identity is so baked into the email protocols that your
Lamson server has almost the inverse problem: it's too easy to fake an
identity, so you need to confirm their subscription and important actions.
This confirmation is traditionally nothing more than a reply asking them to
reply with some random number in the address. If you get a reply to this
auto-generated email address then you assume they are real.
bq. This "random number confirmation handshake" is important because SMTP
allows anyone with an internet connection to lie about who they are and craft a
fake email claiming to be someone else. The confirmation assures you that the
supposed sender has at least received an email in their inbox and that it had
the random number you generated in the reply address. A determined hacker
could also get past that, but if you have that situation you are dealing with
more problems than just confirmations.
Once you do though you are fairly assured that they are identified *and* using
your service. It ends up being the most low effort entry to your service you
could possibly devise without making the service unsafe.
h2. Comparing Two Designs
When you sit down to build a Lamson application you may still think about
registration in the primary way, but a secondary registration is typically more
natural for an email service. To illustrate the point, let's say you are
creating an online game using Lamson, and you want people to register for the
game.
With a primary registration you would approach the problem by making your game
open only to registered users. If they send an email to your game you send a
single reply pointing them at the web page they can use to sign up. You also
then take every email and pass it through a filter that checks that the user is
registered in your subscription database. This is mostly how you would do it
in a web application.
bq. Maybe there is a legitimate reason to force a user to a web page to give a
password and other information before using the service. If that's the case,
then go for it, but consider trying to make the *entry* to your service a
secondary registration that then asks them to do a full registration to get
more features.
With a secondary registration system, you are using Lamson's ability to track
the state of every user to know whether they are registered or not. When your
game receives their first email, it would transition to an @UNKNOWN@ state and
reply with a confirmation email. Once they reply to the confirm email you
transition from @UNKNOWN@ to the rest of the game since they are now
registered. Every interaction after that will simply know they are registered
because they are in the right state.
This has a big impact on your game's design because now you do *not* need to
check the database on every received email to confirm their identity. Their
email address and their state in your state's storage tells you that
implicitly.
You can also extend this to other requirements for your system. You can use
their known state to transition them from free to paid quality services, to
suspended states, to pause them after a bounce, and back to normal activity.
h2. Invite Only Systems
An invite only system can also use the secondary registration technique by
having the inviter's invitation request create a record for the invitee that's
in the right start state. When the inviter tells you to invite their friend,
you create a state for the friend that says @INVITED@ and use the invite as a
sort of implied confirmation email. When the invitee replies to this invite
email, you transition them to the rest of the system as they've now registered.
Another way to think of an invite only system in Lamson is as a 3rd-party
confirmation. Rather than the target user (the invitee) prompting a
confirmation, it is their friend sending the invite (the inviter) who triggers
the confirmation process on behalf of their buddy.
An additional bonus with Lamson is that the state of the invitee is known, so
if the inviter tries to send them again, or if someone else does, then you know
they already were sent an invite. You can just ignore the request and
avoid spamming people. You can also run reports to find out your
conversion rates by simply looking for everyone in the various states and
counting.
h2. Security And Design Issues
Doing these kind of secondary registrations and implied identity does come with
an extra security penalty. Unless your system is incorporating S-MIME or GPG
you won't have a solid way to identify the sender. You don't have this on most
other internet services either so it's not much of a loss. At least with email
you have a higher assurance that this is a real person.
When you design your service it's a good idea to:
# Either make everything the user does non-destructive,
# Send confirmations for anything desctructive,
# Or bounce them to a web site for more complex interactions and security.
You should also make any confirmations you send easily replied to, but have a
good random number in them that you remember for later. In fact, if you use
the "lamson.confirm" API you get all of this for free and done the right way.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/template.html 0000644 0000765 0000024 00000000136 11224460100 024236 0 ustar zedshaw staff
The world is Unicode, but email existed long before that world. Lamson uses
special encoding/decoding gear that takes just about any nasty horrible pre-Unicode
email it gets and converts it to a perfect little Unicode fantasy. You do all
your work in the Unicode world, which Python favors, and then when you’re ready
to send, Lamson intelligently figures out how to make an email that anyone can
read.
The added advantage of doing this conversion is that Lamson by default also
cleans up all the various tricks spammers use to get around spam checkers. Because
it’s doing a conversion from just first principles of “everything must become Unicode”
the end result is a crystal clear piece of data you can filter. When you then
output the same message, it gets reconverted to sane easily parseable message.
Lamson is also so persistent in this conversion that it can convert nearly every
message that’s valid, and the very tiny percentage it can’t (less 1/1000th of a percent)
are entirely screwed up spam that should not be transmitted anyway.
This gear apparently doesn’t support pre-Hindic sanskrit, but then again neither
does Python.
Why
Sometimes people ask why Lamson would bother converting everything to Unicode?
The reason is simple: everything you deal with now is Unicode. Either it’s in
a representation like UTF-8, or it’s internally a Python Unicode string. Databases,
web frameworks, ORM, network protocols, and the entire internet is Unicode.
However, there’s another practical reason. If you were to start using Lamson to
process email, you would end up inventing almost the exact same gear to convert
headers and bodies to Unicode. The only difference is you would do it in a typical
hacker half-assed fashion that only solved your immediate problems, and you wouldn’t
do it in a global useful way.
The Lamson encoding gear is basically what you should be making to deal with email
in a modern language.
How
The lamson.encoding module
is the main meat of this conversion magic. This code is probably the most dense
part of Lamson since it has to use various parsing tricks and heuristics to figure
out how to get every part of the message into a Unicode string.
The primary trick used is that Lamson does not trust anything it’s given, and when
python fails to convert a header or body, it uses the wonderful chardet
library to guess at what the encoding should be based on the contents. If this fails
then the entire data is considered suspect and the message is rejected.
This turns out to be surprisingly accurate in practice, and it even corrects some
invalid clients that fail to encode Subject lines but still place Chinese encodings
in them. Lamson will take those unencoded headers, fail to convert them to ascii,
run chardet on them to see they are actually Chinese, and then convert them accurately.
The Rules
A set of axioms controls how the encoding is done:
NO ENCODING IS TRUSTED, NO LANGUAGE IS SACRED, ALLARESUSPECT.
Python wants Unicode, it will get Unicode.
Any email that CANNOT become Unicode, CANNOT be processed by Lamson or Python.
Email addresses are ESSENTIAL to Lamson’s routing and security, and therefore will be canonicalized and properly encoded.
Lamson will therefore try to “upgrade” all email it receives to Unicode internally, and cleaning all email addresses.
It does this by decoding all codecs, and if the codec LIES, then it will attempt to statistically detect the codec using chardet.
If it can’t detect the codec, and the codec lies, then the email is bad.
All text bodies and attachments are then converted to Python Unicode in the same way as the headers.
All other attachments are converted to raw strings as-is.
Once Lamson has done this, your Python handler can now assume that all
MailRequest objects are happily Unicode enabled and ready to go. The rule is:
IF IT CANNOT BE UNICODE, THENPYTHONCANNOTWORKWITH IT.
On the outgoing end (when you send a MailResponse), Lamson tries to create the
email it wants to receive by canonicalizing it:
All email will be encoded in the simplest cleanest way possible without losing information.
All headers are converted to 'ascii’, and if that doesn’t work, then 'utf-8’.
All text/* attachments and bodies are converted to ascii, and if that doesn’t work, 'utf-8’.
All other attachments are left alone.
All email addresses are normalized and encoded if they have not been already.
Neat Tricks
The end result of this is that you can now take any email and then convert it to
any modern data format and send it over new protocols. For example, here’s the
code from librelist.com to implement the JSON dump
for the archive browser Javascript interface:
def json_encoding(base):
ctype, ctp = base.content_encoding['Content-Type']
cdisp, cdp = base.content_encoding['Content-Disposition']
ctype = ctype or "text/plain"
filename = ctp.get('name',None) or cdp.get('filename', None)
if ctype.startswith('text') or ctype.startswith('message'):
encoding = None
else:
encoding = "base64"
return {'filename': filename, 'type': ctype, 'disposition': cdisp,
'format': encoding}
def json_build(base):
data = {'headers': base.headers,
'body': base.body,
'encoding': json_encoding(base),
'parts': [json_build(p) for p in base.parts],
}
if data['encoding']['format'] and base.body:
data['body'] = base64.b64encode(base.body)
return data
def to_json(base):
return json.dumps(json_build(base), sort_keys=True, indent=4)
Since this code can assume that everything inside Lamson is fully Unicode, it only needs
to worry about binary items like images and encode those as base64.
lamson cleanse
You can also use the lamson cleanse command to take a Maildir or mbox as input, and
then convert it into a cleaned up Maildir as output. Try it on some of your spam
folders to watch the magic.
Reporting Problems
If you happen to run into a message that you feel Lamson should accurately
decode, feel free to report it and I’ll take a look.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/unicode_encoding_and_decoding.txt 0000644 0000765 0000024 00000014431 11243702515 030265 0 ustar zedshaw staff Title: Unicode Encoding And Decoding
The world is Unicode, but email existed long before that world. Lamson uses
special encoding/decoding gear that takes just about any nasty horrible pre-Unicode
email it gets and converts it to a perfect little Unicode fantasy. You do all
your work in the Unicode world, which Python favors, and then when you're ready
to send, Lamson intelligently figures out how to make an email that anyone can
read.
The added advantage of doing this conversion is that Lamson by default also
cleans up all the various tricks spammers use to get around spam checkers. Because
it's doing a conversion from just first principles of "everything must become Unicode"
the end result is a crystal clear piece of data you can filter. When you then
output the same message, it gets reconverted to sane easily parseable message.
Lamson is also so persistent in this conversion that it can convert nearly every
message that's valid, and the very tiny percentage it can't (less 1/1000th of a percent)
are entirely screwed up spam that should not be transmitted anyway.
bq. This gear apparently doesn't support pre-Hindic sanskrit, but then again neither
does Python.
h2. Why
Sometimes people ask why Lamson would bother converting everything to Unicode?
The reason is simple: everything you deal with now is Unicode. Either it's in
a representation like UTF-8, or it's internally a Python Unicode string. Databases,
web frameworks, ORM, network protocols, and the entire internet is Unicode.
However, there's another practical reason. If you were to start using Lamson to
process email, you would end up inventing almost the exact same gear to convert
headers and bodies to Unicode. The only difference is you would do it in a typical
hacker half-assed fashion that only solved your immediate problems, and you wouldn't
do it in a global useful way.
The Lamson encoding gear is basically what you should be making to deal with email
in a modern language.
h2. How
The "lamson.encoding":http://lamsonproject.org/docs/api/lamson.encoding-module.html module
is the main meat of this conversion magic. This code is probably the most dense
part of Lamson since it has to use various parsing tricks and heuristics to figure
out how to get every part of the message into a Unicode string.
The primary trick used is that Lamson does not trust anything it's given, and when
python fails to convert a header or body, it uses the wonderful "chardet":http://chardet.feedparser.org/
library to guess at what the encoding should be based on the contents. If this fails
then the entire data is considered suspect and the message is rejected.
This turns out to be surprisingly accurate in practice, and it even corrects some
invalid clients that fail to encode Subject lines but still place Chinese encodings
in them. Lamson will take those unencoded headers, fail to convert them to ascii,
run chardet on them to see they are actually Chinese, and then convert them accurately.
h2. The Rules
A set of axioms controls how the encoding is done:
# NO ENCODING IS TRUSTED, NO LANGUAGE IS SACRED, ALL ARE SUSPECT.
# Python wants Unicode, it will get Unicode.
# Any email that CANNOT become Unicode, CANNOT be processed by Lamson or Python.
# Email addresses are ESSENTIAL to Lamson's routing and security, and therefore will be canonicalized and properly encoded.
# Lamson will therefore try to "upgrade" all email it receives to Unicode internally, and cleaning all email addresses.
# It does this by decoding all codecs, and if the codec LIES, then it will attempt to statistically detect the codec using chardet.
# If it can't detect the codec, and the codec lies, then the email is bad.
# All text bodies and attachments are then converted to Python Unicode in the same way as the headers.
# All other attachments are converted to raw strings as-is.
Once Lamson has done this, your Python handler can now assume that all
MailRequest objects are happily Unicode enabled and ready to go. The rule is:
IF IT CANNOT BE UNICODE, THEN PYTHON CANNOT WORK WITH IT.
On the outgoing end (when you send a MailResponse), Lamson tries to create the
email it wants to receive by canonicalizing it:
# All email will be encoded in the simplest cleanest way possible without losing information.
# All headers are converted to 'ascii', and if that doesn't work, then 'utf-8'.
# All text/* attachments and bodies are converted to ascii, and if that doesn't work, 'utf-8'.
# All other attachments are left alone.
# All email addresses are normalized and encoded if they have not been already.
h2. Neat Tricks
The end result of this is that you can now take any email and then convert it to
any modern data format and send it over new protocols. For example, here's the
code from "librelist.com":http://librelist.com/ to implement the JSON dump
for the "archive browser":http://librelist.com/browser/ Javascript interface:
def json_encoding(base):
ctype, ctp = base.content_encoding['Content-Type']
cdisp, cdp = base.content_encoding['Content-Disposition']
ctype = ctype or "text/plain"
filename = ctp.get('name',None) or cdp.get('filename', None)
if ctype.startswith('text') or ctype.startswith('message'):
encoding = None
else:
encoding = "base64"
return {'filename': filename, 'type': ctype, 'disposition': cdisp,
'format': encoding}
def json_build(base):
data = {'headers': base.headers,
'body': base.body,
'encoding': json_encoding(base),
'parts': [json_build(p) for p in base.parts],
}
if data['encoding']['format'] and base.body:
data['body'] = base64.b64encode(base.body)
return data
def to_json(base):
return json.dumps(json_build(base), sort_keys=True, indent=4)
Since this code can assume that everything inside Lamson is fully Unicode, it only needs
to worry about binary items like images and encode those as base64.
h2. lamson cleanse
You can also use the @lamson cleanse@ command to take a Maildir or mbox as input, and
then convert it into a cleaned up Maildir as output. Try it on some of your spam
folders to watch the magic.
h2. Reporting Problems
If you happen to run into a message that you feel Lamson should accurately
decode, feel free to "report it":/contact.html and I'll take a look.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/unit_testing.html 0000644 0000765 0000024 00000037540 11243070357 025164 0 ustar zedshaw staff
LamsonProject: Unit Testing
Lamson provides the
lamson.testing
to help with writing unit tests. It doesn’t do everything for you, but does
enough that you can TDD your email interactions with pretend users. It
includes features for checking messages get delivered to queues, checking
spelling, and running things through a fake or real relay.
The Log Server
The first thing you need to do for testing is to run the “log server”:
$ lamson log
The log server acts as the smart relay host you’ve configured in your config/settings.py
file by default. What it does is take all emails that your Lamson application
“sends out” and redeposits them into run/queue. This queue directory is then also
used by lamson.testing for checking that emails were sent out. When anything goes
wrong, you can look in this directory and see what is getting sent out.
An alternative to this setup would be to have all of the sending/routing/relaying done
internal to the whole testing framework, similar to how lamson.testing.RelayConversation works.
However, I found that this made testing that your server actually sends proper emails much
too difficult.
Test Organization
Lamson organizes tests into directories that match the things you’re testing:
librelist $ ls -l tests/
total 16
drwxr-xr-x 6 zedshaw staff 204 Aug 19 17:01 handlers
drwxr-xr-x 8 zedshaw staff 272 Aug 18 15:50 model
drwxr-xr-x 3 zedshaw staff 102 Aug 18 15:50 templates
In most of the projects I’ve only rarely used template tests, but I’ll cover
them below. Model tests make sure that any app.model classes work right, and
handler tests make sure that any app.handler classes work. Nice and simple.
You can add any other directories you want to this, and you can also use
doctests if you want. This comes free with
nosetests and is
very handy.
Handler Tests
We’ll use an example from the librelist.com code base
that validates that a user can subscribe, unsubscribe, and post a message to a
mailing list. This test was primarily written in a TDD style, since generally
interactions and usability testing works better
TDD style.
First you have a common preamble of modules that you need to include:
from nose.tools import *
from lamson.testing import *
from config import settings
import time
from app.model import archive, confirmation
Right away you notice we just include everything from nose.tools and
lamson.testing so that we can use it directly. Yes, this violates Python
style guidelines, but practicality is more important than dogmatic slavery to
supposed standards.
After that we include the config.settings, and two modules from app.model that we’ll
use to check that everything was working.
Notice that we don’t include anything from app.handlers directly. These tests are meant
to be from the perspective of a user interacting with the handler via emails.
Once we have that we do a little setup to clear set some common variables and clear
out some queues we’ll need to check:
Most of these are just variables used in tests later, but the big one is the
client variable. It’s a
lamson.testing.RouterConversation
class that lets you simulate delivering email to your Lamson project.
There’s also a
lamson.testing.TestConversation
class that actually uses your real config/settings.py to connect to a Relay.
This isn’t used so much, but is intended for running “smoke tests” against a
newly deployed server.
With that we’re ready to write out first handler test:
def test_new_user_subscribes_with_invalid_name():
client.begin()
client.say('test-list@%s' % host, "I can't read!", 'noreply')
client.say('test=list@%s' % host, "I can't read!", 'noreply')
clear_queue()
client.say('unbounce@%s' % host, "I have two email addresses!")
assert not delivered('noreply')
assert not delivered('unbounce')
client.say('noreply@%s' % host, "Dumb dumb.")
assert not delivered('noreply')
This is the longest of the tests, and shows all the various things you can do
with the lamson.testing gear. Here’s what we’re doing in order:
Call client.begin to clear out queues and state and start fresh.
Use client.say to send an email from that client to your Lamson application. Notice that you configured the RelayConversation to pretend to be one person with each email getting the same subject line.
Use lamson.testing.clear_queue when you want to make sure the queue is clean.
Use lamson.testing.delivered to check if a certain message from someone is in the queue.
With that you can do pretty much everything you need to send an email and make
sure you get proper replies.
Here’s another example:
def test_new_user_subscribes():
client.begin()
msg = client.say(list_addr, "Hey I was wondering how to fix this?",
list_name + '-confirm')
client.say(msg['Reply-To'], 'Confirmed I am.', 'noreply')
clear_queue()
Notice in this example we have a fourth parameter list_name + '-confirm' and
we get a msg back from our call to client.say. This basically combines
client.say with delivered to do it in one shot. Very commonly, you’ll want
to say something to your server and make sure you got a certain response, and
then do something with that response. This is how you do that.
We then use this '-confirm’ email message to actually subscribe the fake user.
Finally, here’s two more examples:
def test_existing_user_unsubscribes():
test_new_user_subscribes()
msg = client.say(list_name + "-unsubscribe@%s" % host,
"I would like to unsubscribe.", 'confirm')
client.say(msg['Reply-To'], 'Confirmed yes I want out.', 'noreply')
def test_existing_user_posts_message():
test_new_user_subscribes()
msg = client.say(list_addr, "Howdy folks, I was wondering what this is?",
list_addr)
# make sure it gets archived
assert delivered(list_addr, to_queue=queue(queue_path))
In test_existing_user_unsubscribes what we do is call
test_new_user_subscribes to go through that process again, and then we chain
off that to do an unsubscribe. There’s really nothing new here other than that
little trick.
In test_existing_user_posts_message we do the usual send a message and expect
a reply, but then we also make sure that this message was delivered to the
archiver queue.
Apart from those methods and techniques, there’s really nothing more to doing a
handler test. The only additional thing would be using
assert_in_state
to make sure that your handler is in a particular state. I’d recommend against
doing that too much in a handler test, since it will make your tests brittle.
I only do it when the state is very important, such as when checking that they
are in a SPAMMING or BOUNCING state that I need to enforce.
Model Tests
There’s less functionality available in lamson.testing for doing your models. The theory
is that your models will be classes, modules, and ORM that you need to perform the majority
of your storage and analysis. Since has very little to do with email you probably won’t
use lamson.testing as much.
About the only things you might use are APIs for checking that queues get certain messages
in them, and that certain users are in certain states.
Here’s a quick example from librelist.com again that tests how
archives work:
from nose.tools import *
from lamson.testing import *
from lamson.mail import MailRequest, MailResponse
from app.model import archive, mailinglist
import simplejson as json
import shutil
queue_path = archive.store_path('test.list', 'queue')
json_path = archive.store_path('test.list', 'json')
def setup():
clear_queue(queue_path)
shutil.rmtree(json_path)
def teardown():
clear_queue(queue_path)
shutil.rmtree(json_path)
def test_archive_enqueue():
msg = MailResponse(From="zedshaw@zedshaw.com", To="test.list@librelist.com",
Subject="test message", Body="This is a test.")
archive.enqueue('test.list', msg)
assert delivered('zedshaw', to_queue=queue(queue_path))
This is the usual initial setup, and then some extras to make sure that the JSON archives is working. You’ll notice that we hand construct various messages, call methods on the app.model.archive module, and then use delivered to make sure they’re correctly delivered.
Template Tests
Typically you really can only test that your templates are spelled right, or that your templates
render when given certain locals. I’ve found that automated testing of templates isn’t incredibly
useful yet, so the only one I’ve written is from the oneshotblog example:
from nose.tools import *
from lamson.testing import *
from lamson import view
import os
from glob import glob
def test_spelling():
message = {}
original = {}
for path in glob("app/templates/mail/*.msg"):
template = "mail/" + os.path.basename(path)
result = view.render(locals(), template)
spelling(template, result)
This uses lamson.testing.spelling to make sure that each template renders and
that it is spelled correctly. This uses
PyEnchant to do the checking, which
turns out to be rather annoying. If you are interested in improving the
template testing setup, then feel free to talk about your ideas on the lamson
mailing list (but bring code, talk is cheap).
Conclusion
Hopefully you’ll be able to develop your application using good testing techniques with
the lamson.testingAPI. If you find additional testing patterns that could be included
then talk about them on the lamson mailing list to see
if they’re general enough for others.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/unit_testing.txt 0000644 0000765 0000024 00000025251 11243070347 025032 0 ustar zedshaw staff Title: Unit Testing
Lamson provides the
"lamson.testing":http://lamsonproject.org/docs/api/lamson.testing-module.html
to help with writing unit tests. It doesn't do everything for you, but does
enough that you can TDD your email interactions with pretend users. It
includes features for checking messages get delivered to queues, checking
spelling, and running things through a fake or real relay.
h2. The Log Server
The first thing you need to do for testing is to run the "log server":
$ lamson log
The log server acts as the smart relay host you've configured in your @config/settings.py@
file by default. What it does is take all emails that your Lamson application
"sends out" and redeposits them into @run/queue@. This queue directory is then also
used by @lamson.testing@ for checking that emails were sent out. When anything goes
wrong, you can look in this directory and see what is getting sent out.
bq. An alternative to this setup would be to have all of the sending/routing/relaying done
internal to the whole testing framework, similar to how @lamson.testing.RelayConversation@ works.
However, I found that this made testing that your server actually sends proper emails much
too difficult.
h2. Test Organization
Lamson organizes tests into directories that match the things you're testing:
librelist $ ls -l tests/
total 16
drwxr-xr-x 6 zedshaw staff 204 Aug 19 17:01 handlers
drwxr-xr-x 8 zedshaw staff 272 Aug 18 15:50 model
drwxr-xr-x 3 zedshaw staff 102 Aug 18 15:50 templates
In most of the projects I've only rarely used template tests, but I'll cover
them below. Model tests make sure that any @app.model@ classes work right, and
handler tests make sure that any @app.handler@ classes work. Nice and simple.
You can add any other directories you want to this, and you can also use
doctests if you want. This comes free with
"nosetests":http://somethingaboutorange.com/mrl/projects/nose/0.11.1/ and is
very handy.
h2. Handler Tests
We'll use an example from the "librelist.com":http://librelist.com/ code base
that validates that a user can subscribe, unsubscribe, and post a message to a
mailing list. This test was primarily written in a TDD style, since generally
interactions and usability testing works better
"TDD":http://en.wikipedia.org/wiki/Test-driven_development style.
First you have a common preamble of modules that you need to include:
from nose.tools import *
from lamson.testing import *
from config import settings
import time
from app.model import archive, confirmation
Right away you notice we just include everything from @nose.tools@ and
@lamson.testing@ so that we can use it directly. Yes, this violates Python
style guidelines, but practicality is more important than dogmatic slavery to
supposed standards.
After that we include the @config.settings@, and two modules from @app.model@ that we'll
use to check that everything was working.
bq. Notice that we don't include anything from @app.handlers@ directly. These tests are meant
to be from the perspective of a user interacting with the handler via emails.
Once we have that we do a little setup to clear set some common variables and clear
out some queues we'll need to check:
Most of these are just variables used in tests later, but the big one is the
@client@ variable. It's a
"lamson.testing.RouterConversation":http://lamsonproject.org/docs/api/lamson.testing.RouterConversation-class.html
class that lets you simulate delivering email to your Lamson project.
bq. There's also a
"lamson.testing.TestConversation":http://lamsonproject.org/docs/api/lamson.testing.TestConversation-class.html
class that actually uses your real @config/settings.py@ to connect to a Relay.
This isn't used so much, but is intended for running "smoke tests" against a
newly deployed server.
With that we're ready to write out first handler test:
def test_new_user_subscribes_with_invalid_name():
client.begin()
client.say('test-list@%s' % host, "I can't read!", 'noreply')
client.say('test=list@%s' % host, "I can't read!", 'noreply')
clear_queue()
client.say('unbounce@%s' % host, "I have two email addresses!")
assert not delivered('noreply')
assert not delivered('unbounce')
client.say('noreply@%s' % host, "Dumb dumb.")
assert not delivered('noreply')
This is the longest of the tests, and shows all the various things you can do
with the @lamson.testing@ gear. Here's what we're doing in order:
# Call @client.begin@ to clear out queues and state and start fresh.
# Use @client.say@ to send an email from that client to your Lamson application. Notice that you configured the RelayConversation to pretend to be one person with each email getting the same subject line.
# Use @lamson.testing.clear_queue@ when you want to make sure the queue is clean.
# Use @lamson.testing.delivered@ to check if a certain message from someone is in the queue.
With that you can do pretty much everything you need to send an email and make
sure you get proper replies.
Here's another example:
def test_new_user_subscribes():
client.begin()
msg = client.say(list_addr, "Hey I was wondering how to fix this?",
list_name + '-confirm')
client.say(msg['Reply-To'], 'Confirmed I am.', 'noreply')
clear_queue()
Notice in this example we have a fourth parameter @list_name + '-confirm'@ and
we get a @msg@ back from our call to @client.say@. This basically combines
@client.say@ with @delivered@ to do it in one shot. Very commonly, you'll want
to say something to your server and make sure you got a certain response, and
then do something with that response. This is how you do that.
We then use this '-confirm' email message to actually subscribe the fake user.
Finally, here's two more examples:
def test_existing_user_unsubscribes():
test_new_user_subscribes()
msg = client.say(list_name + "-unsubscribe@%s" % host,
"I would like to unsubscribe.", 'confirm')
client.say(msg['Reply-To'], 'Confirmed yes I want out.', 'noreply')
def test_existing_user_posts_message():
test_new_user_subscribes()
msg = client.say(list_addr, "Howdy folks, I was wondering what this is?",
list_addr)
# make sure it gets archived
assert delivered(list_addr, to_queue=queue(queue_path))
In @test_existing_user_unsubscribes@ what we do is call
@test_new_user_subscribes@ to go through that process again, and then we chain
off that to do an unsubscribe. There's really nothing new here other than that
little trick.
In @test_existing_user_posts_message@ we do the usual send a message and expect
a reply, but then we *also* make sure that this message was delivered to the
archiver queue.
Apart from those methods and techniques, there's really nothing more to doing a
handler test. The only additional thing would be using
"assert_in_state":http://lamsonproject.org/docs/api/lamson.testing-module.html#assert_in_state
to make sure that your handler is in a particular state. I'd recommend against
doing that too much in a handler test, since it will make your tests brittle.
I only do it when the state is very important, such as when checking that they
are in a SPAMMING or BOUNCING state that I need to enforce.
h2. Model Tests
There's less functionality available in @lamson.testing@ for doing your models. The theory
is that your models will be classes, modules, and ORM that you need to perform the majority
of your storage and analysis. Since has very little to do with email you probably won't
use @lamson.testing@ as much.
About the only things you might use are APIs for checking that queues get certain messages
in them, and that certain users are in certain states.
Here's a quick example from "librelist.com":http://librelist.com again that tests how
archives work:
from nose.tools import *
from lamson.testing import *
from lamson.mail import MailRequest, MailResponse
from app.model import archive, mailinglist
import simplejson as json
import shutil
queue_path = archive.store_path('test.list', 'queue')
json_path = archive.store_path('test.list', 'json')
def setup():
clear_queue(queue_path)
shutil.rmtree(json_path)
def teardown():
clear_queue(queue_path)
shutil.rmtree(json_path)
def test_archive_enqueue():
msg = MailResponse(From="zedshaw@zedshaw.com", To="test.list@librelist.com",
Subject="test message", Body="This is a test.")
archive.enqueue('test.list', msg)
assert delivered('zedshaw', to_queue=queue(queue_path))
This is the usual initial setup, and then some extras to make sure that the "JSON archives":http://librelist.com/browser/ is working. You'll notice that we hand construct various messages, call methods on the @app.model.archive@ module, and then use @delivered@ to make sure they're correctly delivered.
h2. Template Tests
Typically you really can only test that your templates are spelled right, or that your templates
render when given certain locals. I've found that automated testing of templates isn't incredibly
useful yet, so the only one I've written is from the "oneshotblog":http://oneshotblog.com/ example:
from nose.tools import *
from lamson.testing import *
from lamson import view
import os
from glob import glob
def test_spelling():
message = {}
original = {}
for path in glob("app/templates/mail/*.msg"):
template = "mail/" + os.path.basename(path)
result = view.render(locals(), template)
spelling(template, result)
This uses @lamson.testing.spelling@ to make sure that each template renders and
that it is spelled correctly. This uses
"PyEnchant":http://www.rfk.id.au/software/pyenchant/ to do the checking, which
turns out to be rather annoying. If you are interested in improving the
template testing setup, then feel free to talk about your ideas on "the lamson
mailing list":mailto:lamson@librelist.com (but bring code, talk is cheap).
h2. Conclusion
Hopefully you'll be able to develop your application using good testing techniques with
the @lamson.testing@ API. If you find additional testing patterns that could be included
then "talk about them on the lamson mailing list":mailto:lamson@librelist.com to see
if they're general enough for others.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/writing_a_state_storage.html 0000644 0000765 0000024 00000032554 11242714314 027354 0 ustar zedshaw staff
LamsonProject: Writing A Lamson State Storage Backend
Earlier versions of Lamson assumed that you would use SQLAlchemy, and that you
wouldn’t mind storing your state in the database using SQLAlchemy. Well,
during the 0.9 redesign it became very easy to let you store the state however
and wherever you want. In the new Lamson setup there is a bit more work to
create alternative storage, but Lamson comes with two default stores that you
can use to get started.
The Default MemoryStore
When you get started with Lamson you’ll definitely not want to go through the
trouble of setting up a custom store. For your first days of development,
using the default
MemoryStorage
is the way to go. After you get further in your development you want to switch
to the
ShelveStorage
to store the state in a simple Python
shelve store.
MemoryStorage keeps the routing state in a simple dict in memory, and doesn’t
provide any thread protection. Its purpose is for developer testing and unit
test runs where keeping the state between disks is more of a pain than it is
worth. Use the MemoryStorage (which is the default) for your development
runs and for simple servers where the state does need to be maintained (very
rare).
Here’s the code to MemoryStore for you to just look at, it’s already included in Lamson:
class MemoryStorage(StateStorage):
"""
The default simplified storage for the Router to hold the states. This
should only be used in testing, as you'll lose all your contacts and their
states if your server shutsdown. It is also horribly NOT thread safe.
"""
def __init__(self):
self.states = {}
def get(self, module, sender):
key = self.key(module, sender)
try:
return self.states[key]
except KeyError:
self.set(module, sender, ROUTE_FIRST_STATE)
return ROUTE_FIRST_STATE
def set(self, module, sender, state):
key = self.key(module, sender)
self.states[key] = state
def key(self, module, sender):
return repr([module, sender])
def clear(self):
self.states.clear()
As you can see there isn’t much to implement to make your own storage
for Lamson to use.
Keep in mind that this is just the storage Lamson needs to operate,
you probably don’t want to be accessing this in your own application,
and instead probably want to access the store you create yourself.
The ShelveStorage
ShelveStorage is used for your small deployments where you are mostly just
testing the deployment process or doing a small service. It will store your
data between runs, and is probably fast enough for most sites, but you’ll want
to ditch it if you ever:
Run more than one process that needs the state information.
Start to store everything in a database anyway.
The code to ShelveStorage (which is already part of Lamson) is more complex
since it must keep the threads happy, but you should read through it to get
an idea of how a more complex state store would work:
class ShelveStorage(MemoryStorage):
"""
Uses Python's shelve to store the state of the Routers to disk rather than
in memory like with MemoryStorage. This will get you going on a small
install if you need to persist your states (most likely), but if you
have a database, you'll need to write your own StateStorage that
uses your ORM or database to store. Consider this an example.
"""
def __init__(self, database_path):
"""Database path depends on the backing library use by Python's shelve."""
self.database_path = database_path
self.lock = threading.RLock()
def get(self, module, sender):
"""
This will lock the internal thread lock, and then retrieve from the
shelf whatever module you request. If the module is not found then it
will set (atomically) to ROUTE_FIRST_STATE.
"""
with self.lock:
store = shelve.open(self.database_path)
try:
key = store[self.key(module, sender)]
except KeyError:
self.set(module, sender, ROUTE_FIRST_STATE)
key = ROUTE_FIRST_STATE
return key
def set(self, module, sender, state):
"""
Acquires the self.lock and then sets the requested state in the shelf.
"""
with self.lock:
store = shelve.open(self.database_path)
store[self.key(module, sender)] = state
store.close()
def clear(self):
"""
Primarily used in the debugging/unit testing process to make sure the
states are clear. In production this could be a bad thing.
"""
with self.lock:
store = shelve.open(self.database_path)
store.clear()
store.close()
Using The ShelveStorage
You can use ShelveStorage by simply adding this line to your config/boot.py
file just before you do anything else with the Router:
from lamson.routing import ShelveStorage
Router.STATE_STORE=ShelveStorage("run/states")
It actually doesn’t matter currently when you do it, but it’s good practice right now.
After you do that, restart lamson and it will start using the new store.
Notice that your tests will not use this. It’s not a good idea to have tests
use ShelveStorage, but if you want to turn it on for a run to see what
happens, then you can modify config/testing.py the same way. You could also
write a unit test that did this temporarily by putting that line in your test
case.
What To Implement
If you want to implement your own then you just have to implement the methods
in
StateStorage
and make sure it behaves the same as MemoryStorage. Look at the code to
ShelveStorage for a moment to see what you need:
class StateStorage(object):
"""
The base storage class you need to implement for a custom storage
system.
"""
def get(self, module, sender):
"""
You must implement this so that it returns a single string
of either the state for this combination of arguments, OR
the ROUTE_FIRST_STATE setting.
"""
raise NotImplementedError("You have to implement a StateStorage.get.")
def set(self, module, sender, state):
"""
Set should take the given parameters and consistently set the state for
that combination such that when StateStorage.get is called it gives back
the same setting.
"""
raise NotImplementedError("You have to implement a StateStorage.set.")
def clear(self):
"""
This should clear ALL states, it is only used in unit testing, so you
can have it raise an exception if you want to make this safer.
"""
raise NotImplementedError("You have to implement a StateStorage.clear for unit testing to work.")
There really isn’t much to it, just methods to get and set based on the module
and sender’s email address. Also notice that you don’t have to make it
readable in any complete sense, since Lamson doesn’t do anything other than
get, set, and clear the state store (and it only clears on reloads and in
testing).
Important Considerations
I am purposefully not telling you how to exactly implement it because I’m not
exactly sure what is needed as of the 0.9 release. I use the the
ShelveStorage, but I’d like to hear what other people have written and then
start building infrastructure to make that easier.
There are some important things to consider when you implement your storage
though:
Make sure that the calls to all methods are thread safe, and potentially process safe.
If you do thread locking, use the with statement and an RLock.
If your storage is potentially very slow, then consider a caching scheme inside, but write that after making it work correctly.
Do NOT be tempted to store junk in this like it is a “session”. It should be lean and mean and only do state.
Make sure you keep the key being used exactly as given. You can seriously mess up Lamson’s Router if you start getting fancy.
Attaching It To The Router
Your storage backend will then be attached to the lamson.routing.Router in the
same way as what you did with ShelveStorage. It really should be that simple
since the data stored in the state store is very minimal.
Other Examples
If you want more examples then you can look at the examples/librelist code to see
how librelist.com uses Django
to store the state.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/docs/writing_a_state_storage.txt 0000644 0000765 0000024 00000021466 11242714313 027226 0 ustar zedshaw staff Title: Writing A Lamson State Storage Backend
Earlier versions of Lamson assumed that you would use SQLAlchemy, and that you
wouldn't mind storing your state in the database using SQLAlchemy. Well,
during the 0.9 redesign it became very easy to let you store the state however
and wherever you want. In the new Lamson setup there is a bit more work to
create alternative storage, but Lamson comes with two default stores that you
can use to get started.
h2. The Default MemoryStore
When you get started with Lamson you'll definitely not want to go through the
trouble of setting up a custom store. For your first days of development,
using the default
"MemoryStorage":http://lamsonproject.org/docs/api/lamson.routing.MemoryStorage-class.html
is the way to go. After you get further in your development you want to switch
to the
"ShelveStorage":http://lamsonproject.org/docs/api/lamson.routing.ShelveStorage-class.html
to store the state in a simple Python
"shelve":http://docs.python.org/library/shelve.html store.
@MemoryStorage@ keeps the routing state in a simple dict in memory, and doesn't
provide any thread protection. Its purpose is for developer testing and unit
test runs where keeping the state between disks is more of a pain than it is
worth. Use the @MemoryStorage@ (which is the default) for your development
runs and for simple servers where the state does need to be maintained (very
rare).
Here's the code to MemoryStore for you to just look at, it's already included in Lamson:
class MemoryStorage(StateStorage):
"""
The default simplified storage for the Router to hold the states. This
should only be used in testing, as you'll lose all your contacts and their
states if your server shutsdown. It is also horribly NOT thread safe.
"""
def __init__(self):
self.states = {}
def get(self, module, sender):
key = self.key(module, sender)
try:
return self.states[key]
except KeyError:
self.set(module, sender, ROUTE_FIRST_STATE)
return ROUTE_FIRST_STATE
def set(self, module, sender, state):
key = self.key(module, sender)
self.states[key] = state
def key(self, module, sender):
return repr([module, sender])
def clear(self):
self.states.clear()
As you can see there isn't much to implement to make your own storage
for Lamson to use.
bq. Keep in mind that this is just the storage *Lamson* needs to operate,
you probably don't want to be accessing this in your own application,
and instead probably want to access the store you create yourself.
h2. The ShelveStorage
@ShelveStorage@ is used for your small deployments where you are mostly just
testing the deployment process or doing a small service. It will store your
data between runs, and is probably fast enough for most sites, but you'll want
to ditch it if you ever:
# Run more than one process that needs the state information.
# Start to store everything in a database anyway.
The code to ShelveStorage (which is *already* part of Lamson) is more complex
since it must keep the threads happy, but you should read through it to get
an idea of how a more complex state store would work:
class ShelveStorage(MemoryStorage):
"""
Uses Python's shelve to store the state of the Routers to disk rather than
in memory like with MemoryStorage. This will get you going on a small
install if you need to persist your states (most likely), but if you
have a database, you'll need to write your own StateStorage that
uses your ORM or database to store. Consider this an example.
"""
def __init__(self, database_path):
"""Database path depends on the backing library use by Python's shelve."""
self.database_path = database_path
self.lock = threading.RLock()
def get(self, module, sender):
"""
This will lock the internal thread lock, and then retrieve from the
shelf whatever module you request. If the module is not found then it
will set (atomically) to ROUTE_FIRST_STATE.
"""
with self.lock:
store = shelve.open(self.database_path)
try:
key = store[self.key(module, sender)]
except KeyError:
self.set(module, sender, ROUTE_FIRST_STATE)
key = ROUTE_FIRST_STATE
return key
def set(self, module, sender, state):
"""
Acquires the self.lock and then sets the requested state in the shelf.
"""
with self.lock:
store = shelve.open(self.database_path)
store[self.key(module, sender)] = state
store.close()
def clear(self):
"""
Primarily used in the debugging/unit testing process to make sure the
states are clear. In production this could be a bad thing.
"""
with self.lock:
store = shelve.open(self.database_path)
store.clear()
store.close()
h2. Using The ShelveStorage
You can use @ShelveStorage@ by simply adding this line to your @config/boot.py@
file just before you do anything else with the @Router@:
from lamson.routing import ShelveStorage
Router.STATE_STORE=ShelveStorage("run/states")
It actually doesn't matter currently when you do it, but it's good practice right now.
After you do that, restart lamson and it will start using the new store.
Notice that your *tests will not use this*. It's not a good idea to have tests
use @ShelveStorage@, but if you want to turn it on for a run to see what
happens, then you can modify @config/testing.py@ the same way. You could also
write a unit test that did this temporarily by putting that line in your test
case.
h2. What To Implement
If you want to implement your own then you just have to implement the methods
in
"StateStorage":http://lamsonproject.org/docs/api/lamson.routing.StateStorage-class.html
and make sure it behaves the same as MemoryStorage. Look at the code to
ShelveStorage for a moment to see what you need:
class StateStorage(object):
"""
The base storage class you need to implement for a custom storage
system.
"""
def get(self, module, sender):
"""
You must implement this so that it returns a single string
of either the state for this combination of arguments, OR
the ROUTE_FIRST_STATE setting.
"""
raise NotImplementedError("You have to implement a StateStorage.get.")
def set(self, module, sender, state):
"""
Set should take the given parameters and consistently set the state for
that combination such that when StateStorage.get is called it gives back
the same setting.
"""
raise NotImplementedError("You have to implement a StateStorage.set.")
def clear(self):
"""
This should clear ALL states, it is only used in unit testing, so you
can have it raise an exception if you want to make this safer.
"""
raise NotImplementedError("You have to implement a StateStorage.clear for unit testing to work.")
There really isn't much to it, just methods to get and set based on the module
and sender's email address. Also notice that you don't have to make it
readable in any complete sense, since Lamson doesn't do anything other than
get, set, and clear the state store (and it only clears on reloads and in
testing).
h2. Important Considerations
I am purposefully *not* telling you how to exactly implement it because I'm not
exactly sure what is needed as of the 0.9 release. I use the the
ShelveStorage, but I'd like to hear what other people have written and then
start building infrastructure to make that easier.
There are some important things to consider when you implement your storage
though:
# Make sure that the calls to all methods are thread safe, and potentially process safe.
# If you do thread locking, use the *with* statement and an RLock.
# If your storage is potentially very slow, then consider a caching scheme inside, but *write that after making it work correctly.*
# Do *NOT* be tempted to store junk in this like it is a "session". It should be lean and mean and only do state.
# Make sure you keep the key being used exactly as given. You can seriously mess up Lamson's Router if you start getting fancy.
h2. Attaching It To The Router
Your storage backend will then be attached to the lamson.routing.Router in the
same way as what you did with ShelveStorage. It really should be that simple
since the data stored in the state store is very minimal.
h2. Other Examples
If you want more examples then you can look at the @examples/librelist@ code to see
how "librelist.com":http://librelist.com/ uses "Django":http://www.djangoproject.com/
to store the state.
lamson-1.0pre11/doc/lamsonproject.org/output/download.html 0000644 0000765 0000024 00000012475 11230770143 023323 0 ustar zedshaw staff
LamsonProject: Downloading Lamson
You can get the Lamson source, read the source releases documentation to
find out how.
For people who can’t click the above link, and who still want the source code, do:
bzr branch lp:lamson
But really you should read the read the source releases documentation to
find out how.
Installing With Easy Install
Using easy_install is the easiest way to install, simply run “sudo easy_install lamson” and
it will do the work of getting the dependencies and setting everything up.
Installing With setup.py
Once you get the source, you can also use setup.py to install Lamson. First make sure that
you have Mako, and nose installed:
If you refuse to use easy_install entirely then it is on you to find these projects and
install them how you see best. *Make sure you have the most recent version of all
dependencies.*
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/download.txt 0000644 0000765 0000024 00000002413 11227340123 023162 0 ustar zedshaw staff From: Zed
Title: Downloading Lamson
You can get the Lamson *source*, read "the source releases":/releases/ documentation to
find out how.
For people who can't click the above link, and who still want the source code, do:
bzr branch lp:lamson
But really you should read the read "the source releases":/releases/ documentation to
find out how.
h2. Installing With Easy Install
Using easy_install is the easiest way to install, simply run "sudo easy_install lamson" and
it will do the work of getting the dependencies and setting everything up.
h2. Installing With setup.py
Once you get the source, you can also use setup.py to install Lamson. First make sure that
you have *Mako*, and *nose* installed:
If you refuse to use easy_install entirely then it is on you to find these projects and
install them how you see best. *Make sure you have the most recent version of all
dependencies.*
Then you can use this command to install:
$ python setup.py install
h2. Installing Into A Virtual Env
Read the instructions "for installing with virtual env":/docs/lamson_virtual_env.html in the "documentation":/docs/ section.
lamson-1.0pre11/doc/lamsonproject.org/output/favicon.ico 0000644 0000765 0000024 00000000000 11203537656 022736 0 ustar zedshaw staff lamson-1.0pre11/doc/lamsonproject.org/output/feed.xml 0000644 0000765 0000024 00000062744 11313464306 022262 0 ustar zedshaw staff
Lamson Project(TM)http://lamsonproject.orgLamson The Python SMTP Server and FrameworkSun, 20 Dec 2009 10:11:18 GMTPyRSS2Gen-1.0.0http://blogs.law.harvard.edu/tech/rssSmall Lamson 1.0pre10 Updatehttp://lamsonproject.org/blog/2009-12-13.html<h2><a href="/blog/2009-12-13.html">2009-12-13</a> : Small Lamson 1.0pre10 Update</h2>
<p> <p>There was a very minor one line fix that corrects a fairly obnoxious bug
in Lamson due to my not understanding how <i>nonzero</i> works.</p></p>
<p><a href="/blog/2009-12-13.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-12-13.htmlSun, 20 Dec 2009 10:05:30 GMTLamson 1.0pre9 Is Outhttp://lamsonproject.org/blog/2009-12-12.html<h2><a href="/blog/2009-12-12.html">2009-12-12</a> : Lamson 1.0pre9 Is Out</h2>
<p> <p><strong>UPDATE</strong>: This used to be an announce for 1.0pre8, but python-daemon apparently has a bug that makes it not honor
the <span class="caps">TERM</span> signal (still) so I had to go back to <span class="caps">HUP</span>.</p></p>
<p><a href="/blog/2009-12-12.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-12-12.htmlSun, 20 Dec 2009 10:05:30 GMTQuick Lamson 1.0pre6 Release, More This Weekendhttp://lamsonproject.org/blog/2009-09-26.html<h2><a href="/blog/2009-09-26.html">2009-09-26</a> : Quick Lamson 1.0pre6 Release, More This Weekend</h2>
<p> <p>I put up a quick release of Lamson that fixes a bug in the gen project prototype.
I recently switched all the domain names being used in the test suite to “localhost” or
a similar nonexistant domain name. Why? Well, because some people liked to run
the Lamson test suite or their fresh project test suite against a <strong>live</strong> <span class="caps">SMTP</span>
server. The test suite would “send” mail to test@test.com and my own
address, under the assumption that people would use the <code>lamson log</code> test server.</p></p>
<p><a href="/blog/2009-09-26.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-09-26.htmlSun, 20 Dec 2009 10:05:30 GMTLamson 1.0pre5 Outhttp://lamsonproject.org/blog/2009-09-07.html<h2><a href="/blog/2009-09-07.html">2009-09-07</a> : Lamson 1.0pre5 Out</h2>
<p> <p>I just pushed Lamson 1.0pre5 to PyPI for your enjoyment:</p></p>
<p><a href="/blog/2009-09-07.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-09-07.htmlSun, 20 Dec 2009 10:05:30 GMTLamson 1.0pre4 Out, Lots Of Docs Done, 100% Coveragehttp://lamsonproject.org/blog/2009-08-22.html<h2><a href="/blog/2009-08-22.html">2009-08-22</a> : Lamson 1.0pre4 Out, Lots Of Docs Done, 100% Coverage</h2>
<p> <p>I happy to announce probably one of the last few releases before
I officially put the 1.0 stamp on Lamson. This last 1% of the things
I want to do takes a while, but it really puts a good shine on the
project.</p></p>
<p><a href="/blog/2009-08-22.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-08-22.htmlSun, 20 Dec 2009 10:05:30 GMTLamson 1.0pre2, HTML Email, Standalonehttp://lamsonproject.org/blog/2009-08-03.html<h2><a href="/blog/2009-08-03.html">2009-08-03</a> : Lamson 1.0pre2, HTML Email, Standalone</h2>
<p> <p>Lamson 1.0pre2 features two features that might signal the end of the beginning or
the beginning of the end, depending on your perspective: <span class="caps">HTML</span> Email and Lamson Standalone.
<span class="caps">HTML</span> Email support comes from a new module
<a href="http://lamsonproject.org/docs/api/lamson.html-module.html">lamson.html</a> that gives a nice template
method to send out <span class="caps">HTML</span> to victims…uh…customers. Lamson Standalone will be a way
to run Lamson as your customized email server instead of another server like Postfix.</p></p>
<p><a href="/blog/2009-08-03.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-08-03.htmlSun, 20 Dec 2009 10:05:30 GMTLamson 1.0pre1 Releasedhttp://lamsonproject.org/blog/2009-07-20.html<h2><a href="/blog/2009-07-20.html">2009-07-20</a> : Lamson 1.0pre1 Released</h2>
<p> <p>Tonight I’m releasing Lamson 1.0pre1 with all the latest improvements I’ve made while
making <a href="http://librelist.com/">librelist.com</a> and taking feedback from a few people
using Lamson. The goal from now on will be to basically squash bugs and write docs, with
only a rare feature or two as I find them needed on projects.</p></p>
<p><a href="/blog/2009-07-20.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-07-20.htmlSun, 20 Dec 2009 10:05:30 GMTI Blame Bounces For The Delayhttp://lamsonproject.org/blog/2009-07-19.html<h2><a href="/blog/2009-07-19.html">2009-07-19</a> : I Blame Bounces For The Delay</h2>
<p> <p>I was going to release Lamson 0.9.6 much earlier than this, but as I worked on
the bounce detection feature, I realized that if I just wrote that and a few
other cleanups and features, I’d actually have a <strong>1.0pre1</strong> candidate on my
hands instead of a 0.9.6 release.</p></p>
<p><a href="/blog/2009-07-19.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-07-19.htmlSun, 20 Dec 2009 10:05:30 GMTLamson 0.9.6 Sometime Todayhttp://lamsonproject.org/blog/2009-07-14.html<h2><a href="/blog/2009-07-14.html">2009-07-14</a> : Lamson 0.9.6 Sometime Today</h2>
<p> <p>I think I’m at a point where bounce detection and handling works really well and is
ready for release. I’ve got it running on the latest Lamson demo <a href="http://librelist.com/">librelist.com</a>
and it works. I’ll be pushing out a new release of Libre List in a little while
that does some advanced bounce handling, and if that all works, then I’ll push
out 0.9.6.</p></p>
<p><a href="/blog/2009-07-14.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-07-14.htmlSun, 20 Dec 2009 10:05:30 GMTLamson's Bounce Detection Algorithmhttp://lamsonproject.org/blog/2009-07-09.html<h2><a href="/blog/2009-07-09.html">2009-07-09</a> : Lamson's Bounce Detection Algorithm</h2>
<p> <p>I just finished committing 0.9.6 code for doing bounce detection and analysis
with Lamson. It’s part of the new mailing list example I’m coding up for the
0.9.6 release which I’ll be running on a free mailing list site I’m going
to release soon. In this blog post I’d like to go through the bounce detection
algorithm and get some feedback and samples from people. So far it works great for
the samples I have, but I want it to be fairly bullet proof.</p></p>
<p><a href="/blog/2009-07-09.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-07-09.htmlSun, 20 Dec 2009 10:05:30 GMTLamson Mailing Lists Downhttp://lamsonproject.org/blog/2009-07-07.html<h2><a href="/blog/2009-07-07.html">2009-07-07</a> : Lamson Mailing Lists Down</h2>
<p> <p>I’m making up the release of Lamson and doing some server maintenance so I took down the
Lamson server running the mailing lists. I’ll let everyone know when they’re back up.</p></p>
<p><a href="/blog/2009-07-07.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-07-07.htmlSun, 20 Dec 2009 10:05:30 GMTArticle In The Reg About Lamson (By Ted Dziuba)http://lamsonproject.org/blog/2009-07-03.html<h2><a href="/blog/2009-07-03.html">2009-07-03</a> : Article In The Reg About Lamson (By Ted Dziuba)</h2>
<p> <p>Just a quick update to point people at an article in <a href="http://www.theregister.co.uk/">The Register</a> by
<a href="http://teddziuba.com/">Ted Dziuba</a> entitled <a href="http://www.theregister.co.uk/2009/07/03/lamson/">Lamson – email app coding without the palm sweat: Doing what Java never did</a>. He interviewed me about Lamson, things I think you can use it for, and other fun stuffs.</p></p>
<p><a href="/blog/2009-07-03.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-07-03.htmlSun, 20 Dec 2009 10:05:30 GMTLamson 0.9.5, The Push To 1.0http://lamsonproject.org/blog/2009-06-26.html<h2><a href="/blog/2009-06-26.html">2009-06-26</a> : Lamson 0.9.5, The Push To 1.0</h2>
<p> <p>I just released Lamson 0.9.5 with all the major Unicode refactoring done and working.
This is an important release because 0.9.5 is where I declare that I’m pushing to a 1.0
release and the base Lamson <a href="/docs/api/">APIs</a> won’t change under penalty of death. In fact they can’t
change because I’m using them myself in a few applications.</p></p>
<p><a href="/blog/2009-06-26.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-06-26.htmlSun, 20 Dec 2009 10:05:30 GMT0.9.5 Almost There, But Stumped On Templateshttp://lamsonproject.org/blog/2009-06-22.html<h2><a href="/blog/2009-06-22.html">2009-06-22</a> : 0.9.5 Almost There, But Stumped On Templates</h2>
<p> <p>Since the 0.9.4 release I’ve rewritten the main part of the decoding parser
so that it’s much cleaner and handles more edge conditions. If there’s one
word that defines what makes <span class="caps">MIME</span> horrible it would be “edge”.</p></p>
<p><a href="/blog/2009-06-22.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-06-22.htmlSun, 20 Dec 2009 10:05:30 GMTLamson 0.9.4 With Unicode Super Powershttp://lamsonproject.org/blog/2009-06-20.html<h2><a href="/blog/2009-06-20.html">2009-06-20</a> : Lamson 0.9.4 With Unicode Super Powers</h2>
<p> <p>Lamson 0.9.4 is out and it’s sporting a completely rewritten and meticulously
crafted encoding system. With the new <a href="http://lamsonproject.org/docs/api/lamson.encoding-module.html">lamson.encoding</a> code Lamson can now decode nearly any nasty horrible encoded spam
or mail you hand it, turn it into pristine nice Python unicode strings, and
then output sweet clean ascii or utf-8 in a consistent way.</p></p>
<p><a href="/blog/2009-06-20.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-06-20.htmlSun, 20 Dec 2009 10:05:30 GMTThe Mailocalypse Is Upon Us!http://lamsonproject.org/blog/2009-06-14.html<h2><a href="/blog/2009-06-14.html">2009-06-14</a> : The Mailocalypse Is Upon Us!</h2>
<p> <p>I’m currently polishing off the two final features before I start going for
the Lamson 1.0 release. I’ve been using Lamson to make a few little cute applications
and create one thing for a potential client, and so far I haven’t had to change much
since 0.9.3. It’s great so far and I hope that Lamson 1.0 will be a fun release.</p></p>
<p><a href="/blog/2009-06-14.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-06-14.htmlSun, 20 Dec 2009 10:05:30 GMTLamson At NYLUG Python Workshop Today @ 6:00PMhttp://lamsonproject.org/blog/2009-06-09.html<h2><a href="/blog/2009-06-09.html">2009-06-09</a> : Lamson At NYLUG Python Workshop Today @ 6:00PM</h2>
<p> <p>Just a quick reminder that I’ll be presenting <a href="http://lamsonproject.org/">Lamson</a> to the
<span class="caps">NYLUG</span> Python Workshop today at 6:00PM. The event is at
the NY Public Library Hudson Park Branch, 66 Leroy St., NY NY 10014
in <span class="caps">NYC</span> and you can <a href="http://nylug.org/pipermail/nylug-announce/2009-June/000795.html">find out more here.</a></p></p>
<p><a href="/blog/2009-06-09.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-06-09.htmlSun, 20 Dec 2009 10:05:30 GMTA Screencast And Docs On Deploying Lamson And OneShotBloghttp://lamsonproject.org/blog/2009-06-08.html<h2><a href="/blog/2009-06-08.html">2009-06-08</a> : A Screencast And Docs On Deploying Lamson And OneShotBlog</h2>
<p> <p>I just finished writing some new documentation on <a href="/docs/deploying_lamson.html">Deploying Lamson and OneShotBlog</a> that
shows you how to install Lamson and all required software into a completely clean Python 2.6 and virtualenv configuration.
The instructions take you from nothing to a running <a href="http://oneshotblog.com/">oneshotblog.com</a> installation that you can
play with and hack on. The process of doing Lamson deployments is still a little too rough for me, but these instructions
should get people started.</p></p>
<p><a href="/blog/2009-06-08.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-06-08.htmlSun, 20 Dec 2009 10:05:30 GMTLamson 0.9.3 Is Out And Sexy As Hellhttp://lamsonproject.org/blog/2009-06-06.html<h2><a href="/blog/2009-06-06.html">2009-06-06</a> : Lamson 0.9.3 Is Out And Sexy As Hell</h2>
<p> <p>This release is the result of me working on my little
<a href="http://oneshotblog.com/">oneshotblog.com</a> project while tweaking and refining
Lamson as I go. The end result is 0.9.3 didn’t have a lot of big code changes,
but all the tiny little changes add up to a very nice release. The highlights
of this release are more secure server runs, better character encoding handling
for headers, various cleanups in how mail is queued, and fixes for Python 2.6
support.</p></p>
<p><a href="/blog/2009-06-06.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-06-06.htmlSun, 20 Dec 2009 10:05:30 GMTOneShotBlog Sample (Hack) Runninghttp://lamsonproject.org/blog/2009-06-04.html<h2><a href="/blog/2009-06-04.html">2009-06-04</a> : OneShotBlog Sample (Hack) Running</h2>
<p> <p>I <strong>finally</strong> got off my ass and put the <a href="http://oneshotblog.com/">OneShotBlog</a> sample up.
This is the code (plus a few little tweaks) from the sample that is in the <a href="/releases/">Lamson source</a>
running on another server. It’s using all the features of Lamson, include the new <a href="/docs/deferred_processing_to_queues.html">Queue Receiver</a>
functionality. So far it’s working great considering I’ve just been hacking on it on the side to try
out the usability of the Lamson <a href="/docs/api/">APIs</a> and not really taken it seriously.</p></p>
<p><a href="/blog/2009-06-04.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-06-04.htmlSun, 20 Dec 2009 10:05:30 GMTLamson 0.9.1 Out, New Docshttp://lamsonproject.org/blog/2009-06-03.html<h2><a href="/blog/2009-06-03.html">2009-06-03</a> : Lamson 0.9.1 Out, New Docs</h2>
<p> <p>I released Lamson 0.9.1 today so please <a href="/download.html">grab it and test it</a> and shoot me feedback.</p></p>
<p><a href="/blog/2009-06-03.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-06-03.htmlSun, 20 Dec 2009 10:05:30 GMTLamson 0.9.2, Test Coverage 97%http://lamsonproject.org/blog/2009-06-03-2.html<h2><a href="/blog/2009-06-03-2.html">2009-06-03-2</a> : Lamson 0.9.2, Test Coverage 97%</h2>
<p> <p>The 0.9.2 release is out and ready for everyone to easy_install. I spent the
day getting rid of my tech debt by boosting the Lamson test coverage to a
whopping 97%.</p></p>
<p><a href="/blog/2009-06-03-2.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-06-03-2.htmlSun, 20 Dec 2009 10:05:30 GMTLamson 0.9 Is Out, Find My Bugs!http://lamsonproject.org/blog/2009-06-01.html<h2><a href="/blog/2009-06-01.html">2009-06-01</a> : Lamson 0.9 Is Out, Find My Bugs!</h2>
<p>
I just pushed Lamson 0.9 up to PyPI for everyone to grab and break. This release
features a complete redesign of the routing, state handling, templating, and a full
set of very complete documentation. Everyone who was using 0.8.x series should be able
to migrate to this version with some work, but it won’t be terribly painful (assuming
you have unit tests).</p>
<p><a href="/blog/2009-06-01.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-06-01.htmlSun, 20 Dec 2009 10:05:30 GMTLamson 0.9 Later Todayhttp://lamsonproject.org/blog/2009-05-31.html<h2><a href="/blog/2009-05-31.html">2009-05-31</a> : Lamson 0.9 Later Today</h2>
<p> <p>I have been working hard on the documentation and scrubbing the code for Lamson and the 0.9 release
coming out soon. The only things I feel I need to do before an official 0.9 release are:</p></p>
<p><a href="/blog/2009-05-31.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-05-31.htmlSun, 20 Dec 2009 10:05:30 GMT0.9-pre2 Up For Testing, Docs Toohttp://lamsonproject.org/blog/2009-05-28.html<h2><a href="/blog/2009-05-28.html">2009-05-28</a> : 0.9-pre2 Up For Testing, Docs Too</h2>
<p> <p>First off, my apologies to everyone if your <span class="caps">RSS</span> reader went crazy today.
I include documentation changes in the <span class="caps">RSS</span> feed so that people can
easily track updates to the Lamson docs. However, that means when I’m
writing a lot of documentation it hits the feed repeatedly.</p></p>
<p><a href="/blog/2009-05-28.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-05-28.htmlSun, 20 Dec 2009 10:05:30 GMTFeatures For The 0.9 Release (Soon)http://lamsonproject.org/blog/2009-05-24.html<h2><a href="/blog/2009-05-24.html">2009-05-24</a> : Features For The 0.9 Release (Soon)</h2>
<p>
<p>I've been hard at work cooking up the very nice new routing system, and
I must say it is rather tasty. I've gone and created a whole new
routing and state management design that uses decorators right in your
handler modules to indicate how each state will expect mail addresses.
</p></p>
<p><a href="/blog/2009-05-24.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-05-24.htmlSun, 20 Dec 2009 10:05:30 GMTLamson Project Ideashttp://lamsonproject.org/blog/2009-05-20.html<h2><a href="/blog/2009-05-20.html">2009-05-20</a> : Lamson Project Ideas</h2>
<p> <p>I wrote a <a href="http://zedshaw.com/blog/2009-05-20.html">blog post about project ideas for Lamson</a>
on my personal blog. Head on over if you’re looking for something to hack on, or just
want something to read that isn’t about the web.</p></p>
<p><a href="/blog/2009-05-20.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-05-20.htmlSun, 20 Dec 2009 10:05:30 GMTNew Site Look, Same Great Contenthttp://lamsonproject.org/blog/2009-05-19.html<h2><a href="/blog/2009-05-19.html">2009-05-19</a> : New Site Look, Same Great Content</h2>
<p> <p>This is just a quick update to say thanks to <a href="http://kenkeiter.com/">Ken Keiter</a>
for creating a new <a href="http://lamsonproject.org/">lamsonproject.org</a> site layout and
design. The new site should be easier to read, have more breathing room, and look
easier on the eyes. It’s even got a logo:</p></p>
<p><a href="/blog/2009-05-19.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-05-19.htmlSun, 20 Dec 2009 10:05:30 GMTBug Fix 0.8.4, Mailing Lists, Spam Blockinghttp://lamsonproject.org/blog/2009-05-18.html<h2><a href="/blog/2009-05-18.html">2009-05-18</a> : Bug Fix 0.8.4, Mailing Lists, Spam Blocking</h2>
<p> <p>A few announcements from my work on Lamson the last few days. I managed to fix
a bug, put Lamson to work doing Lamson’s mailing lists, and use Lamson to do
some spam blocking on my own email account.</p></p>
<p><a href="/blog/2009-05-18.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-05-18.htmlSun, 20 Dec 2009 10:05:30 GMTLamson Project Site Launchedhttp://lamsonproject.org/blog/2009-05-16.html<h2><a href="/blog/2009-05-16.html">2009-05-16</a> : Lamson Project Site Launched</h2>
<p> <p>Today I launched the Lamson Project site at
<a href="http://lamsonproject.org/">lamsonproject.org</a> and started filling in the
content. Lamson is really turning into a fun and useful project, and hopefully
the site will get other people interested in it and using it.</p></p>
<p><a href="/blog/2009-05-16.html">Read more...</a></p>
http://lamsonproject.org/blog/2009-05-16.htmlSun, 20 Dec 2009 10:05:30 GMT lamson-1.0pre11/doc/lamsonproject.org/output/home_template.html 0000644 0000765 0000024 00000012317 11230770103 024326 0 ustar zedshaw staff
LamsonProject: $title
We've all been there, mucking around in the sendmail m4 macros trying one more time to get the damn mailing list to update for the new users. Every time we say, "This sucks, I want to rewrite this stupid thing." Yet, when we're done, we simply crawl back to our caves covered in our sendmail wounds.
Lamson's goal is to put an end to the hell that is "e-mail application development". Rather than stay stuck in the 1970s, Lamson adopts modern web application framework design and uses a proven scripting language (Python).
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
We've all been there, mucking around in the sendmail m4 macros trying one more time to get the damn mailing list to update for the new users. Every time we say, "This sucks, I want to rewrite this stupid thing." Yet, when we're done, we simply crawl back to our caves covered in our sendmail wounds.
Lamson's goal is to put an end to the hell that is "e-mail application development". Rather than stay stuck in the 1970s, Lamson adopts modern web application framework design and uses a proven scripting language (Python).
Lamson The Python Mail Server
Rather than stay stuck in the 1970's, Lamson adopts modern web application
framework design and uses a proven scripting language (Python).
Instead of hideous aliases files (that you never remember to update)
Lamson uses friendly regular expressions and routing decorators that make
configuring how email is handled trivial.
Rather than bizarre flat file "databases" and hashtable stores, Lamson uses anything
that Python does to store data. You can use all of the following to store data in Lamson:
There's so many great ways to store data in Python that Lamson doesn't make
any assumptions other than to provide a convenience function or two for configuring
SQLAlchemy (since it's kind of a pain to setup). In reality, there's so many
ORMs and storage mechanisms available to Python that you should try as many as
you can and use the one you like the most.
Lamson also supports extensive spam blocking through SpamBayes:
All within a framework that works well with existing legacy mail servers so you can
gradually get into using Lamson.
The 30 Second Introduction
If you have Python and easy_install already, then try this out:
$ easy_install lamson
$ lamson gen -project mymailserver
$ cd mymailserver
$ lamson start
$ lamson log
$ nosetests
$ lamson help -for send
$ lamson send -sender me@mydomain.com -to test@test.com \
-subject "My test." -body "Hi there." -port 8823
$ less logs/lamson.log
$ mutt -F muttrc
$ lamson stop -ALL run
You now have a working base Lamson setup ready for you to work on with everything you need installed.
Next Steps
Grab the code and you can read through the
quick start documentation. After you've gone through
that, best thing to do is read the code to the examples/osb example included in
the source distribution and read the rest of the
documentation on this site.
At any point, you can get help for all available Lamson commands with:
$ lamson help
You can get individual command help with:
$ lamson help -for start
Finally, if you really want to get started using Lamson to implement your dream email
application, but are completely lost, then you can contact me for help.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/index.txt 0000644 0000765 0000024 00000010721 11227340344 022470 0 ustar zedshaw staff From: Zed
Title: Lamson The Python Mail Server
template: input/home_template.html
Content-Type: text/html
Rather than stay stuck in the 1970's, Lamson adopts modern web application
framework design and uses a proven scripting language (Python).
Instead of hideous aliases files (that you never remember to update)
Lamson uses friendly regular expressions and routing decorators that make
configuring how email is handled trivial.
Rather than bizarre flat file "databases" and hashtable stores, Lamson uses anything
that Python does to store data. You can use all of the following to store data in Lamson:
There's so many great ways to store data in Python that Lamson doesn't make
any assumptions other than to provide a convenience function or two for configuring
SQLAlchemy (since it's kind of a pain to setup). In reality, there's so many
ORMs and storage mechanisms available to Python that you should try as many as
you can and use the one you like the most.
Lamson also supports extensive spam blocking through SpamBayes:
All within a framework that works well with existing legacy mail servers so you can
gradually get into using Lamson.
The 30 Second Introduction
If you have Python and easy_install already, then try this out:
$ easy_install lamson
$ lamson gen -project mymailserver
$ cd mymailserver
$ lamson start
$ lamson log
$ nosetests
$ lamson help -for send
$ lamson send -sender me@mydomain.com -to test@test.com \
-subject "My test." -body "Hi there." -port 8823
$ less logs/lamson.log
$ mutt -F muttrc
$ lamson stop -ALL run
You now have a working base Lamson setup ready for you to work on with everything you need installed.
Next Steps
Grab the code and you can read through the
quick start documentation. After you've gone through
that, best thing to do is read the code to the examples/osb example included in
the source distribution and read the rest of the
documentation on this site.
At any point, you can get help for all available Lamson commands with:
$ lamson help
You can get individual command help with:
$ lamson help -for start
Finally, if you really want to get started using Lamson to implement your dream email
application, but are completely lost, then you can contact me for help.
Lamson’s mailing lists are hosted on librelist.com
which is also written in Lamson.
You can subscribe to the list by sending an email to lamson@librelist.com
with your first message. Remember that librelist posts this message, so introduce yourself.
If you want to read the archives for the Lamson list, you can browse them
here in a very very basic
form (soon to improve).
You can also get the archives for the Lamson list by using rsync:
Be careful when you do this since that will get everything. It’s better
to figure out which month or day you want using your web browser and
then get that directory only.
IRC Channel
For people who don’t want to join the mailing list but still need help
there’s a #lamson irc channel on irc.freenode.org. Come by and ask
your question, and stick around since it might take people a little
while to respond.
See the download instructions for information on getting lamson, and read the getting started instructions to start your own application in less than 10 minutes.
Mailing Lists
Lamson hosts its own mailing lists as well as provides a free open mailing list
service for anyone who needs one. Simply send an email to the list you want @librelist.com and it will
get you started.
lamson-1.0pre11/doc/lamsonproject.org/output/lists/index.txt 0000644 0000765 0000024 00000002111 11310754212 023614 0 ustar zedshaw staff From: Zed
Title: Lamson Project Mailing Lists
Lamson's mailing lists are hosted on "librelist.com":http://librelist.com/
which is also written in Lamson.
You can subscribe to the list by sending an email to "lamson@librelist.com":mailto:lamson@librelist.com
with your first message. *Remember that librelist posts this message, so introduce yourself.*
If you want to read the archives for the Lamson list, you can browse them
"here":http://librelist.com/browser/ in a very very basic
form (soon to improve).
You can also get the archives for the Lamson list by using rsync:
Be careful when you do this since that will get everything. It's better
to figure out which month or day you want using your web browser and
then get that directory only.
h2. IRC Channel
For people who don't want to join the mailing list but still need help
there's a #lamson irc channel on irc.freenode.org. Come by and ask
your question, and stick around since it might take people a little
while to respond.
lamson-1.0pre11/doc/lamsonproject.org/output/mailocalypse.py 0000644 0000765 0000024 00000005344 11215271227 023661 0 ustar zedshaw staff import email
from email.header import make_header, decode_header
from string import capwords
import sys
import mailbox
ALL_MAIL = 0
BAD_MAIL = 0
def all_parts(msg):
parts = [m for m in msg.walk() if m != msg]
if not parts:
parts = [msg]
return parts
def collapse_header(header):
if header.strip().startswith("=?"):
decoded = decode_header(header)
converted = (unicode(
x[0], encoding=x[1] or 'ascii', errors='replace')
for x in decoded)
value = u"".join(converted)
else:
value = unicode(header, errors='replace')
return value.encode("utf-8")
def convert_header_insanity(header):
if header is None:
return header
elif type(header) == list:
return [collapse_header(h) for h in header]
else:
return collapse_header(header)
def encode_header(name, val, charset='utf-8'):
msg[name] = make_header([(val, charset)]).encode()
def bless_headers(msg):
# go through every header and convert it to utf-8
headers = {}
for h in msg.keys():
headers[capwords(h, '-')] = convert_header_insanity(msg[h])
return headers
def dump_headers(headers):
for h in headers:
print h, headers[h]
def mail_load_cleanse(msg_file):
global ALL_MAIL
global BAD_MAIL
msg = email.message_from_file(msg_file)
headers = bless_headers(msg)
# go through every body and convert it to utf-8
parts = all_parts(msg)
bodies = []
for part in parts:
guts = part.get_payload(decode=True)
if part.get_content_maintype() == "text":
charset = part.get_charsets()[0]
try:
if charset:
uguts = unicode(guts, part.get_charsets()[0])
guts = uguts.encode("utf-8")
else:
guts = guts.encode("utf-8")
except UnicodeDecodeError, exc:
print >> sys.stderr, "CONFLICTED CHARSET:", exc, part.get_charsets()
BAD_MAIL += 1
except LookupError, exc:
print >> sys.stderr, "UNKNOWN CHARSET:", exc, part.get_charsets()
BAD_MAIL += 1
except Exception, exc:
print >> sys.stderr, "WEIRDO ERROR", exc, part.get_charsets()
BAD_MAIL += 1
ALL_MAIL += 1
mb = None
try:
mb = mailbox.Maildir(sys.argv[1])
len(mb) # need this to make the maildir try to read the directory and fail
except OSError:
print "NOT A MAILDIR, TRYING MBOX"
mb = mailbox.mbox(sys.argv[1])
if not mb:
print "NOT A MAILDIR OR MBOX, SORRY"
for key in mb.keys():
mail_load_cleanse(mb.get_file(key))
print >> sys.stderr, "ALL", ALL_MAIL
print >> sys.stderr, "BAD", BAD_MAIL
lamson-1.0pre11/doc/lamsonproject.org/output/prettify.css 0000644 0000765 0000024 00000001346 11213057477 023212 0 ustar zedshaw staff /* Pretty printing styles. Used with prettify.js. */
.str { color: #ed9d13; }
.kwd { color: #6ab825; }
.com { color: #999999; font-style: italic; }
.typ { color: #447fcf; }
.lit { color: #cd2828; }
.pun { color: #e2e2e2; }
.pln { color: #f2f2f2; }
.tag { color: #cd2828; }
.atn { color: #f2f2f2; }
.atv { color: #f2f2f2; }
.dec { color: #f2f2f2; }
pre.prettyprint { padding: 2px; border: 1px solid #888; }
@media print {
.str { color: #060; }
.kwd { color: #006; font-weight: bold; }
.com { color: #600; font-style: italic; }
.typ { color: #404; font-weight: bold; }
.lit { color: #044; }
.pun { color: #440; }
.pln { color: #000; }
.tag { color: #006; font-weight: bold; }
.atn { color: #404; }
.atv { color: #060; }
}
lamson-1.0pre11/doc/lamsonproject.org/output/._prettify.js 0000644 0000765 0000024 00000000311 11213050674 023232 0 ustar zedshaw staff Mac OS X 2 ATTR H3 1 1 com.apple.quarantine q/0000;4a2c5065;Firefox.app;|org.mozilla.firefox lamson-1.0pre11/doc/lamsonproject.org/output/prettify.js 0000644 0000765 0000024 00000153247 11213050674 023036 0 ustar zedshaw staff // Copyright (C) 2006 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/**
* @fileoverview
* some functions for browser-side pretty printing of code contained in html.
*
* The lexer should work on a number of languages including C and friends,
* Java, Python, Bash, SQL, HTML, XML, CSS, Javascript, and Makefiles.
* It works passably on Ruby, PHP and Awk and a decent subset of Perl, but,
* because of commenting conventions, doesn't work on Smalltalk, Lisp-like, or
* CAML-like languages.
*
* If there's a language not mentioned here, then I don't know it, and don't
* know whether it works. If it has a C-like, Bash-like, or XML-like syntax
* then it should work passably.
*
* Usage:
* 1) include this source file in an html page via
*
* 2) define style rules. See the example page for examples.
* 3) mark the
