We Recommend

Learning Python
The authors of Learning Python show you enough essentials of the Python scripting language to enable you to begin solving problems right away, then reveal more powerful aspects of the language one at a time. This approach is sure to appeal to programmers and system administrators who have urgent problems and a preference for learning by semi-guided experimentation.

Posted By

mustam on 08/27/09

Tagged

twitter logging

Versions (?)

Who likes this?

1 person has marked this snippet as a favorite

gpupo

Backup my twitter-log

Published in: Python

e.g.

$ twlog.py user
twitter account name: user
Now logging start, take several times. r u OK? [y/N]: y
DEBUG: make dir: twitter-log-user
DEBUG: get log: page=1
DEBUG: new tweet: 21, accum. tweet: 21
DEBUG: remain hits: 149
DEBUG: write file: 20090828-105421-page0001.xml
DEBUG: wait 3sec.
...
DEBUG: get log: page=7
DEBUG: new tweet: 0, accum. tweet: 115
DEBUG: remain hits: 143
OK!

Highlighted TextExpand

# vim: ts=2 sw=2 expandtab
import sys
from os import mkdir
from os.path import exists
import time
from urllib2 import urlopen
from xml.dom import Node
from xml.dom.minidom import parseString
from time import strptime, strftime, localtime, time, sleep
from calendar import timegm

debug=True

def getRemain():
  url='http://twitter.com/account/rate_limit_status.xml'
  raw=urlopen(url).read()
  dom=parseString(raw)
  tag=dom.getElementsByTagName('remaining-hits').item(0)
  remain=int(tag.firstChild.data)
  return remain

def getResetTime():
  url='http://twitter.com/account/rate_limit_status.xml'
  raw=urlopen(url).read()
  dom=parseString(raw)
  tag=dom.getElementsByTagName('reset-time').item(0)
  resetTime=tag.firstChild.data
  return resetTime

def getIds(dom):
  ids=[]
  for idNode in dom.getElementsByTagName('id'):
    ids.append(int(idNode.childNodes[0].data))
  return ids

def getPage(user, page):
  url='http://twitter.com/statuses/user_timeline/'
  rawData=urlopen(url+user+'.xml?page='+str(page)).read()
  return rawData

def getEpoch(str):
	dd = strptime(str, "%a %b %d %H:%M:%S +0000 %Y")
	return timegm(dd)

def addEpochAttr(dom):
	ds=dom.getElementsByTagName('created_at')
	for d in ds:
		dtext=d.firstChild.data
		depoch=getEpoch(dtext)
		d.setAttribute('epoch', str(depoch))

def debug(str):
  if debug: print 'DEBUG: '+str

### MAIN ####################################################
if(len(sys.argv)<2):
  print 'Usage: '+sys.argv[0]+' <twitter account name> [page offset]'
else:
  user=sys.argv[1]
  print 'twitter account name: '+user
  dir='twitter-log-'+user
  sleepTime=3
  
  # CONFIRM
  s=raw_input('Now logging start, take several times. r u OK? [y/N]: ')
  
  if s.lower()=='y' or s.lower()=='yes':
    # START LOGGING
    remain=getRemain()
    if(remain>0):
      if(not exists(dir)): mkdir(dir) # make dir
      debug('make dir: '+dir)
      idSet=set()                           # init
      if(len(sys.argv)>2): page=int(sys.argv[2])
      else:                page=1
      pageStr='%04d'%page
      rawData=getPage(user, page)           # get log
      dom=parseString(rawData)
      addEpochAttr(dom)
      debug('get log: page='+str(page))
      newIdSet=set(getIds(dom))-idSet
      idSet=idSet|newIdSet
      debug('new tweet: '+str(len(newIdSet))+', accum. tweet: '+str(len(idSet)))
      remain=getRemain()
      debug('remain hits: '+str(remain))
  
      while(len(newIdSet)>0 and remain>0):
        rawData=dom.toxml("utf-8")
        path=strftime("%Y%m%d-%H%M%S", localtime(time()))+'-page'+pageStr+'.xml'
        f=open(dir+'/'+path, 'w')
        f.write(rawData)
        debug('write file: '+path)
  
        debug('wait '+str(sleepTime)+'sec.')
        sleep(sleepTime);                   # wait
  
        page+=1
        pageStr='%04d'%page
        rawData=getPage(user, page)           # get log
        dom=parseString(rawData)
        addEpochAttr(dom)
        debug('get log: page='+str(page))
        newIdSet=set(getIds(dom))-idSet
        idSet=idSet|newIdSet
        debug('new tweet: '+str(len(newIdSet))+', accum. tweet: '+str(len(idSet)))
        remain=getRemain()
        debug('remain hits: '+str(remain))

if(remain==0):
        print 'Error: no more request.'
        print 'Next reset time(UTC): '+getResetTime()
      else: print 'OK!'

else:
      print 'Error: no more request.'
      print 'Next reset time(UTC): '+getResetTime()
  else: print 'Aborted.'

Report this snippet

Comment:

You need to login to post a comment.