Skis / Toys / Fun

If you have a few minutes and speak python & twisted, it would be useful to have an extra set of eyes on this section of code. The basic idea of this is to be a reconnecting thrift client, such that I can just write simple client.function(a,b,c) calls without having to worry about if there is or isn’t a client and it will queue reconnect as needed.

from thrift.transport import TTwisted
from thrift.protocol import TBinaryProtocol
from twisted.internet.protocol import ReconnectingClientFactory
from twisted.internet import defer, reactor
from twisted.python import failure
from collections import deque
from redback import log
 
class ClientBusy(Exception):
    pass
 
class ClientDead(Exception):
    pass
 
class InvalidThriftRequest(Exception):
    pass
 
class ManagedThriftRequest(object):
    def __init__(self, method, *args, **kw) :
        self.method = method 
        self.args   = args
        self.kw     = kw
 
class ManagedClient(object) :
    def __init__(self, factory) :
        self.__factory = factory
 
    def _is_connected(self) :
        return self.__factory.is_connected()
 
    def __getattr__(self, name) :
        if hasattr(self.__factory.client_class, name) :
            def f(*args, **kw) :
                return self.__factory.pushRequest(ManagedThriftRequest(name, *args, **kw))
            return f
        raise InvalidThriftRequest("Cant find method: %s" % name)
 
class ManagedThriftClientProtocol(TTwisted.ThriftClientProtocol):
    def __init__(self, client_class, iprot_factory, oprot_factory=None):
        TTwisted.ThriftClientProtocol.__init__(self, client_class, iprot_factory, oprot_factory)
        self.client_class = client_class
        self.deferred = None
        self.alive    = False
 
    def connectionMade(self):
        log.debug(self, "Connection made to", "%s.%s" % (self.client_class.__module__, self.client_class.__name__), "[%s]:%s" % self.transport.addr)
        self.alive = True
        TTwisted.ThriftClientProtocol.connectionMade(self)
        self.client.protocol = self
        self.factory.clientIdle(self)
 
    def connectionLost(self, reason=None):
        log.debug(self, "Connection lost to", "%s.%s" % (self.client_class.__module__, self.client_class.__name__), "[%s]:%s" % self.transport.addr)
        self.alive = False
        try :
            TTwisted.ThriftClientProtocol.connectionLost(self, reason)
            self.factory.clientGone(self)
        except Exception, e :
            log.error(self.connectionLost, e)
 
    def _complete(self, res, request, dfd):
        self.deferred = None
        if isinstance(res, failure.Failure) and dfd :
            self.factory.pushRequest(request, dfd)
        else :
            if dfd :
                dfd.callback(res)
            self.factory.clientIdle(self)
        return res
 
    def submitRequest(self, request, dfd):
        if not self.alive :
            raise ClientBusy
        if not self.deferred :
            fun = getattr(self.client, request.method, None)
            if not fun:
                raise InvalidThriftRequest("No such method as : %s" % request.method)
            else :
                try :
                    d = fun(*request.args, **request.kw)
                except Exception, e :
                    log.error(self.submitRequest, "calling : ", request.method, e)
            self.deferred = d
            d.addBoth(self._complete, request, dfd)
            return d
        else:
            raise ClientBusy
 
class ManagedClientFactory(ReconnectingClientFactory):
    maxDelay        = 5
    thriftFactory   = TBinaryProtocol.TBinaryProtocolAcceleratedFactory
    protocol        = ManagedThriftClientProtocol
    submitLoopSleep = 0
    client_class    = None
 
    def __init__(self, client_class=None):
        self._stack        = deque()
        self._protos       = defer.DeferredQueue()
        self.deferred      = defer.Deferred()
        self.client_class  = client_class or self.client_class
        self.client        = ManagedClient(self)
 
    def _errback(self, reason=None):
        if self.deferred :
            self.deferred.errback(reason)
            self.deferred = None
 
    def _callback(self, value=None):
        if self.deferred :
            self.deferred.callback(value)
            self.deferred = None
 
    def clientConnectionFailed(self, connector, reason):
        ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)
        self._errback(reason)
 
    def buildProtocol(self, addr) :
        self.resetDelay()
        p = self.protocol(self.client_class, self.thriftFactory())
 
        p.factory = self
        #self._protos.put(p)
        return p
 
    def clientIdle(self, proto) :
        self._callback(True)
        if proto.alive :
            self._protos.put(proto)
 
    def clientGone(self, proto):
        pass
        #if proto in self._protos :
        #    self._protos.remove(proto)
 
    def _protoErr(self, proto):
        pass
        #import traceback
        #traceback.print_stack()
        #print "IN PROTO ERR", proto
 
    def _protoReady(self, proto):
        if proto.deferred : 
            log.msg(self._protoReady, "Proto currently active!")
            return
        if not proto.alive : 
            log.msg(self._protoReady, "Proto currently dead!")
            return
 
        try:
            request, deferred = self._stack.popleft()
        except defer.QueueUnderflow :
            pass
 
        d = proto.submitRequest(request, deferred)
        return d
 
    def pushRequest(self, request, din=None) :
        d = din or defer.Deferred()
        self._stack.append((request, d))
 
        dfd = self._protos.get()
        dfd.addCallback(self._protoReady)
        dfd.addErrback(self._protoErr)
        return d
 
    def shutdown(self) :
        """Shutdown this factory"""
 
        self.stopTrying()
        for p in self._protos:
            if p.transport:
                p.transport.loseConnection()

Tags:   python · thrift · twisted

I’ve had lots of idea of things to build, as somebody once said ideas are a dime a dozen, it’s doing it that’s the hard work.  Between talking to companies over the years, or just random things I’ve stumbled across there has to be a good idea to put more energy into.

Things I’ve built in the last few years — in some state of “goodness” (90% complete or better).

• totallypsp.com - Basic idea was to be a daily featured marketplace for PSP games and accessories.   It’s there it’s running, but once it was going to move from a “build” to manage step I ran out of interest… [100% done]
• zapquiz.com - Originally came out of a need to have a way to make some review quizes for my PSIA exams.  The core idea was to have a community of people contribute questions on a subject and they have people take sample tests and work forward.   My daughter wants me to finish it, but I’m not sure what “finished” looks like.  This is a case where I’ve not figured out what the product is fully… or how to make it FUN. [95% done]
• notewave.com - Shared drawings in the vein of sketchfu.com or other shared drawing services.  This half in my mind morphed into focusing on creative clipart on photos… Which yeilded a non-existant iPhone app which would allow you to take a photo, mark-it up and share with your friends on facebook.  The facebook side of things is working where you can create wallposts with photos from your albums and creative drawing, though the UI is crude and it’s only avalable if you have a secret AppID…  [95% done]
• IrisWall – my Windows Flickr screen saver — it was finished, just never released… Mentioned it to a friend the other day and he was like “Cool Can I get it…” hmm.. source code is in some dusty folder… [100% done]
• feedini.com – Yet another news agregator, I think this space is dead again…  Had some cool ideas, but not worth anything [90% done].
• A CMS for google app engine, it allowed for uploading of zip files as template bundles…  But, the problem is that it’s a total pain to configure appengine for 3rd party applications.  The idea was that a small business could use the CMS as the base for a website — with all the CMS features (WYSIWIG, etc.) and a template collection ala wordpress to quickly get an online presence.   [90% done]

Now things I’ve been thinking about:

• a CRM for job seekers — though I’ve now found about three others, not sure if it’s a worthwhile space.
• Combining the idea of short status updates and 360′ reviews into a tool that allows you to have a shared vision of how you’re performing and what you’re working on.   Image if a tool periodically pinged you or your co-workers about what you were doing and your performance/perception…
• Simple portal pages – think kosmix.com, but with the baggage of a search engine.  Focus in on domain name parking and companies like that where potentially providing a more delightful experience around a subject area might increase the parked domain experience.
• Tools to extract structured information from webpages in a way similar to Yahoo!s Search Monkey but utilzing some better document techniques to provid automated infrastructure.  I keep on having vauge visions of a meeting a quack.com from 10 years ago.  They probably had the best system I’ve seen to date, but yet there are still some huge things to do.

Fundamentally, I want to build things to delight users.   It’s not too hard to get passionate about a problem, but fundamentally I want to really know who I’m building for…  Heck, it doesn’t even have to be “hard” — just has to be something that people find annoying and want improved.

What are your pain points – or where should I spend my energy?

Tags:

Just a random idea that came out of a strange hallway conversation on the cost of health care. Here’s the idea of the week.

One of the current big problems in America is the over consumption of food yielding both health problems and just waste. Since we’re currently trying to figure out how to pay for our healthcare system the quick idea is to tax calories in the style of other sin taxes. Think of it this way, a typical person consumes 2000 calories a day, if there was a sin tax of $0.20 (e.g 1 cent per 100 calories) it would be a small price to pay. The quick observation is that if you’re eating a Bacon Double Cheeseburger with fries and a large coke you’re probably just at a 2000 calorie lunch. So now when you eat breakfast and dinner you’re clearly contributing to your long term health care plan… that emergency room visit for a coronary bypass.

The quick modification is that there really should be no tax on any fresh or frozen produce. If you really were eating half of your calories in produce clearly your health care costs are going to be lower, than Mr. Bacon Dbl Cheeseburger.

Tags:   healthcare · tax

By allowing selection in a UI can lead to a huge impact in what you expect — not really rocket science… In an app that only allows for the addition (+undo for removal) your mental model is fairly simple. The moment you add the concept of selection all of a sudden your expectations jump exponentially. Think of the differences between Paint and Draw. In a paint app you don’t really think about a whole class of operations, but in your draw app all of a sudden you feel like you could move where a brush stroke is.

It’s really funny how drawing a little blue box around something to indicate selection, has such an enormous change.

Tags:

Had a great discussion this morning with a co-worker.  The crux of the conversation is Google vs. Facebook — who is going to win. Is Google Wave the game changer that will kill facebook?

Nope.  We’ve really created three different product spaces in these organizations.  We’ve got Google which is “defined” as an applications company (gmail, docs, apps, etc.) and facebook which is social.

Can Google beat Facebook or Facebook beat Google — probably not — I don’t quite see how my “who’s got prettier eyes”  news feed item has any meaning on my “apps” page on iGoogle.  Sure, in a tab, etc, etc.  But, at the same time should facebook be a news feed, platform, or ???  Fundamentally, I don’t see why facebook isn’t pushing it’s platform harder and harder than Google.   It would be nice to see every site offer a login with facebook, let me move my data my personality around the web.  Make my friends part of my web experience, cut down on the “do you want to give permission” gunk.  Go for two-tier access — anonymous FB vs. logged in FB.

That way when I click on the “who’s eyes are these” trivial quiz, I can participate without sharing all of my data, but then they can impose the “join” functionality when I feel like it.

Side note — the funny part is that facebook is great for accepted bi-directional friendship, but twitter proves there’s a solid marketplace for uni-directional friendship.  Which is handled — weakly — by “fan” pages on Facebook.   It’s probably a topic unto itself to discuss how business can utilize Twitter/Facebook to market their “fan” tendancies to deliver new customers.

Tags:   facebook · google · rant

This idea came up in a conversation the other day with somebody… The crux of it is that more startups, toy websites or other research projects have been created over the years to aggregate blogs, twitters, lives into a single stream. Everything from facebook, friendfeed, google reader, netvibes, etc. Hey, even I wrote one (feedini.com — it’s probably not running at the moment).

The problem is that none of these experiences can compare to a newspaper in a few ways:

• Focus
  We’ve all read and re-read the same story over 37 different blogs or news outlets, which gets to be a tedious pain.
• Breadth
  If I’m interested in “food” I’m interested in food, reading a single blog isn’t going to make me feel like I’m reading the food section of the newspaper.
• Diversity
  I don’t read the sports section, but ever now and then I find that because it’s there I see something that’s worth reading.

This got me thinking, everybody has been trying to make a NLP/Machine learning system that creates the “right” content for you.  What happens if that’s the wrong idea!

My quick proposal, is that somebody should create an editors desk.  Think of it as bloglines meets about.com — I can create a personal newspaper that I can focus and drive anyway I want….  Create editorial content or republish content quickly easily, etc.   With the added benefit that I could format it similar to a news site (rather than a blog).

This way my readership could look at a collection of stories that I was publishing — technically republishing (think AP wire).  I could establish the editorial voice of the site and have conversations/discussions that we close to the readership (think Hacker News).  Then make sure there is a solid set of editor tools — this is where my inner geek get off, this is where the NLP/Machine learning could help an editor focus in on important content to republish on their site.

Thus if you wanted to make a food portal you create “food.example.com” select a layout from the inventory — ala bloglines templates — then feed it a collection of interesting feeds and it would suggest a bunch more that were simlar… Walla you’re now 80% of the way to making and editorial portal.

Maybe it’s just democratizing Huffington Post…

Tags:   agregation · feeds · nlp